def test_fit_pSMAC(self): output = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC') self._setUp(output) X_train, Y_train, X_test, Y_test = putil.get_dataset('iris') automl = AutoSklearnClassifier(time_left_for_this_task=15, per_run_time_limit=15, output_folder=output, tmp_folder=output, shared_mode=True, seed=1, initial_configurations_via_metalearning=0, ensemble_size=0) automl.fit(X_train, Y_train) # Create a 'dummy model' for the first run, which has an accuracy of # more than 99%; it should be in the final ensemble if the ensemble # building of the second AutoSklearn classifier works correct true_targets_ensemble_path = os.path.join(output, '.auto-sklearn', 'true_targets_ensemble.npy') true_targets_ensemble = np.load(true_targets_ensemble_path) true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0 probas = np.zeros((len(true_targets_ensemble), 3), dtype=float) for i, value in enumerate(true_targets_ensemble): probas[i, value] = 1.0 dummy_predictions_path = os.path.join(output, '.auto-sklearn', 'predictions_ensemble', 'predictions_ensemble_1_00030.npy') with open(dummy_predictions_path, 'wb') as fh: np.save(fh, probas) probas_test = np.zeros((len(Y_test), 3), dtype=float) for i, value in enumerate(Y_test): probas_test[i, value] = 1.0 dummy = ArrayReturningDummyPredictor(probas_test) backend = Backend(output, output) backend.save_model(dummy, 30, 1) automl = AutoSklearnClassifier(time_left_for_this_task=15, per_run_time_limit=15, output_folder=output, tmp_folder=output, shared_mode=True, seed=2, initial_configurations_via_metalearning=0, ensemble_size=0) automl.fit(X_train, Y_train) automl.run_ensemble_builder(0, 1, 50).wait() score = automl.score(X_test, Y_test) self.assertEqual(len(os.listdir(os.path.join(output, '.auto-sklearn', 'ensembles'))), 1) self.assertGreaterEqual(score, 0.90) self.assertEqual(automl._task, MULTICLASS_CLASSIFICATION) del automl self._tearDown(output)
def test_fit_pSMAC(self): output = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC') self._setUp(output) X_train, Y_train, X_test, Y_test = putil.get_dataset('iris') automl = AutoSklearnClassifier(time_left_for_this_task=15, per_run_time_limit=15, output_folder=output, tmp_folder=output, shared_mode=True, seed=1, initial_configurations_via_metalearning=0, ensemble_size=0) automl.fit(X_train, Y_train) # Create a 'dummy model' for the first run, which has an accuracy of # more than 99%; it should be in the final ensemble if the ensemble # building of the second AutoSklearn classifier works correct true_targets_ensemble_path = os.path.join(output, '.auto-sklearn', 'true_targets_ensemble.npy') true_targets_ensemble = np.load(true_targets_ensemble_path) true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0 probas = np.zeros((len(true_targets_ensemble), 3), dtype=float) for i, value in enumerate(true_targets_ensemble): probas[i, value] = 1.0 dummy_predictions_path = os.path.join(output, '.auto-sklearn', 'predictions_ensemble', 'predictions_ensemble_1_00030.npy') with open(dummy_predictions_path, 'wb') as fh: np.save(fh, probas) probas_test = np.zeros((len(Y_test), 3), dtype=float) for i, value in enumerate(Y_test): probas_test[i, value] = 1.0 dummy = ArrayReturningDummyPredictor(probas_test) backend = Backend(output, output) backend.save_model(dummy, 30, 1) automl = AutoSklearnClassifier(time_left_for_this_task=15, per_run_time_limit=15, output_folder=output, tmp_folder=output, shared_mode=True, seed=2, initial_configurations_via_metalearning=0, ensemble_size=0) automl.fit(X_train, Y_train) automl.run_ensemble_builder(0, 1, 50).wait() score = automl.score(X_test, Y_test) self.assertEqual(len(os.listdir(os.path.join(output, '.auto-sklearn', 'ensemble_indices'))), 1) self.assertGreaterEqual(score, 0.90) self.assertEqual(automl._task, MULTICLASS_CLASSIFICATION) del automl self._tearDown(output)
def test_file_output(self): shutil.rmtree(self.working_directory, ignore_errors=True) os.mkdir(self.working_directory) queue_mock = unittest.mock.Mock() context = BackendContext( temporary_directory=os.path.join(self.working_directory, 'tmp'), output_directory=os.path.join(self.working_directory, 'out'), delete_tmp_folder_after_terminate=True, delete_output_folder_after_terminate=True, ) with unittest.mock.patch.object( Backend, 'load_datamanager') as load_datamanager_mock: load_datamanager_mock.return_value = get_multiclass_classification_datamanager( ) backend = Backend(context) ae = AbstractEvaluator( backend=backend, output_y_hat_optimization=False, queue=queue_mock, metric=accuracy, port=self.port, ) ae.model = sklearn.dummy.DummyClassifier() rs = np.random.RandomState() ae.Y_optimization = rs.rand(33, 3) predictions_ensemble = rs.rand(33, 3) predictions_test = rs.rand(25, 3) predictions_valid = rs.rand(25, 3) ae.file_output( Y_optimization_pred=predictions_ensemble, Y_valid_pred=predictions_valid, Y_test_pred=predictions_test, ) self.assertTrue( os.path.exists( os.path.join(self.working_directory, 'tmp', '.auto-sklearn', 'runs', '1_0_None'))) shutil.rmtree(self.working_directory, ignore_errors=True)
def test_refit_shuffle_on_fail(self): output = os.path.join(self.test_dir, '..', '.tmp_refit_shuffle_on_fail') context = BackendContext(output, output, False, False) backend = Backend(context) failing_model = unittest.mock.Mock() failing_model.fit.side_effect = [ValueError(), ValueError(), None] auto = AutoML(backend, 20, 5) ensemble_mock = unittest.mock.Mock() auto.ensemble_ = ensemble_mock ensemble_mock.get_model_identifiers.return_value = [1] auto.models_ = {1: failing_model} X = np.array([1, 2, 3]) y = np.array([1, 2, 3]) auto.refit(X, y) self.assertEqual(failing_model.fit.call_count, 3)
def test_add_additional_components(self): shutil.rmtree(self.working_directory, ignore_errors=True) os.mkdir(self.working_directory) queue_mock = unittest.mock.Mock() context = BackendContext( temporary_directory=os.path.join(self.working_directory, 'tmp'), delete_tmp_folder_after_terminate=True, ) with unittest.mock.patch.object( Backend, 'load_datamanager') as load_datamanager_mock: load_datamanager_mock.return_value = get_multiclass_classification_datamanager( ) backend = Backend(context) with unittest.mock.patch.object(_addons['classification'], 'add_component') as _: # If the components in the argument `additional_components` are an empty dict # there is no call to `add_component`, if there's something in it, `add_component # is called (2nd case) for fixture, case in ((0, dict()), (1, dict(abc='def'))): thirdparty_components_patch = unittest.mock.Mock() thirdparty_components_patch.components = case additional_components = dict( classification=thirdparty_components_patch) AbstractEvaluator( backend=backend, output_y_hat_optimization=False, queue=queue_mock, metric=accuracy, port=self.port, additional_components=additional_components, ) self.assertEqual( _addons['classification'].add_component.call_count, fixture)
def test_fit_pSMAC(self): tmp = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC') output = os.path.join(self.test_dir, '..', '.out_estimator_fit_pSMAC') self._setUp(tmp) self._setUp(output) X_train, Y_train, X_test, Y_test = putil.get_dataset('digits') # test parallel Classifier to predict classes, not only indexes Y_train += 1 Y_test += 1 automl = AutoSklearnClassifier( time_left_for_this_task=20, per_run_time_limit=5, output_folder=output, tmp_folder=tmp, shared_mode=True, seed=1, initial_configurations_via_metalearning=0, ensemble_size=0, ) automl.fit(X_train, Y_train) # Create a 'dummy model' for the first run, which has an accuracy of # more than 99%; it should be in the final ensemble if the ensemble # building of the second AutoSklearn classifier works correct true_targets_ensemble_path = os.path.join(tmp, '.auto-sklearn', 'true_targets_ensemble.npy') with open(true_targets_ensemble_path, 'rb') as fh: true_targets_ensemble = np.load(fh) true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0 true_targets_ensemble = true_targets_ensemble.astype(int) probas = np.zeros((len(true_targets_ensemble), 10), dtype=float) for i, value in enumerate(true_targets_ensemble): probas[i, value] = 1.0 dummy_predictions_path = os.path.join( tmp, '.auto-sklearn', 'predictions_ensemble', 'predictions_ensemble_1_00030.npy', ) with open(dummy_predictions_path, 'wb') as fh: np.save(fh, probas) probas_test = np.zeros((len(Y_test), 10), dtype=float) for i, value in enumerate(Y_test): probas_test[i, value - 1] = 1.0 dummy = ArrayReturningDummyPredictor(probas_test) context = BackendContext(tmp, output, False, False, True) backend = Backend(context) backend.save_model(dummy, 30, 1) automl = AutoSklearnClassifier( time_left_for_this_task=20, per_run_time_limit=5, output_folder=output, tmp_folder=tmp, shared_mode=True, seed=2, initial_configurations_via_metalearning=0, ensemble_size=0, ) automl.fit_ensemble( Y_train, task=MULTICLASS_CLASSIFICATION, metric=accuracy, precision='32', dataset_name='iris', ensemble_size=20, ensemble_nbest=50, ) predictions = automl.predict(X_test) score = sklearn.metrics.accuracy_score(Y_test, predictions) self.assertEqual( len(os.listdir(os.path.join(tmp, '.auto-sklearn', 'ensembles'))), 1) self.assertGreaterEqual(score, 0.90) self.assertEqual(automl._automl._task, MULTICLASS_CLASSIFICATION) models = automl._automl.models_ classifier_types = [type(c) for c in models.values()] self.assertIn(ArrayReturningDummyPredictor, classifier_types) del automl self._tearDown(tmp) self._tearDown(output)
def main(task_id, ensemble_dir, performance_range_threshold, ensemble_size, max_keep_best, seed, only_portfolio_runs, call_from_cmd): if max_keep_best > 1: assert max_keep_best == int(max_keep_best) max_keep_best = int(max_keep_best) memory_limit = 4000 precision = 32 metric = make_scorer('balanced_accuracy_fast', BalancedAccuracy()) if not os.path.exists(ensemble_dir): raise NotADirectoryError("%s does not exist") if call_from_cmd: assert str(task_id) in ensemble_dir fl_name = "ensemble_results_%fthresh_%dsize_%fbest" % \ (performance_range_threshold, ensemble_size, max_keep_best) if only_portfolio_runs: fl_name += "_only_portfolio" fl_name = os.path.join(ensemble_dir, fl_name) if os.path.isfile(fl_name): raise ValueError("Nothing left to do, %s already exists" % fl_name) # figure out how many prediction files are in dir if call_from_cmd: pred_dir = os.path.join(ensemble_dir, "auto-sklearn-output", ".auto-sklearn", "predictions_ensemble") n_models = glob.glob(pred_dir + "/predictions_ensemble_%d_*.npy.gz" % seed) else: pred_dir = os.path.join(ensemble_dir, ".auto-sklearn", "predictions_ensemble") n_models = glob.glob(pred_dir + "/predictions_ensemble_%d_*.npy" % seed) n_models.sort(key=lambda x: int(float(x.split("_")[-2]))) print("\n".join(n_models)) print("Found %d ensemble predictions" % len(n_models)) if len(n_models) == 0: raise ValueError("%s has no ensemble predictions" % pred_dir) # Get start time of ensemble building: 1) load json 2) find key 3) get creation times if call_from_cmd: timestamps_fl = os.path.join(ensemble_dir, "auto-sklearn-output", "timestamps.json") else: timestamps_fl = os.path.join(ensemble_dir, "timestamps.json") with open(timestamps_fl, "r") as fh: timestamps = json.load(fh) model_timestamps = None overall_start_time = None for k in timestamps: if "predictions_ensemble" in k: model_timestamps = timestamps[k] if "start_time_%d" % seed in timestamps[k]: overall_start_time = timestamps[k]["start_time_%d" % seed] timestamp_keys = list(model_timestamps.keys()) for timestamp_key in timestamp_keys: if timestamp_key.endswith( 'lock') or 'predictions_ensemble' not in timestamp_key: del model_timestamps[timestamp_key] assert model_timestamps is not None and overall_start_time is not None assert len(model_timestamps) == len(n_models), (len(model_timestamps), len(n_models)) # Get overall timelimit vanilla_results_fl = os.path.join(ensemble_dir, "result.json") with open(vanilla_results_fl, "r") as fh: vanilla_results = json.load(fh) # If only portfolio configurations, read runhistory if only_portfolio_runs: if call_from_cmd: runhistory_fl = os.path.join(ensemble_dir, "auto-sklearn-output", "smac3-output", "run*", "runhistory.json") else: runhistory_fl = os.path.join(ensemble_dir, "smac3-output", "run*", "runhistory.json") runhistory_fl = glob.glob(runhistory_fl) assert len(runhistory_fl) == 1 with open(runhistory_fl[0], "r") as fh: runhistory = json.load(fh) init_design_num_runs = [] for i in runhistory["data"]: if i[1][3]["configuration_origin"] == "Initial design": if "error" in i[1][3]: continue init_design_num_runs.append(i[1][3]["num_run"]) print("Portfolio stopped after %s runs" % str(init_design_num_runs)) last_run = max(init_design_num_runs) print("Cut down to only portfolio runs fom %d" % len(n_models)) for i, n in enumerate(n_models): if int(float(n.split("_")[-2])) > last_run: n_models = n_models[:i] break print("... to %d" % len(n_models)) # load data X_train, y_train, X_test, y_test, cat = load_task(task_id) if len(np.unique(y_test)) == 2: task_type = BINARY_CLASSIFICATION elif len(np.unique(y_test)) > 2: task_type = MULTICLASS_CLASSIFICATION else: raise ValueError("Unknown task type for task %d" % task_id) tmp_dir = tempfile.TemporaryDirectory() loss_trajectory = [] # Construct ensemble builder context = BackendContextMock( temporary_directory=(ensemble_dir + "/auto-sklearn-output/" if call_from_cmd else ensemble_dir), output_directory=tmp_dir.name, delete_tmp_folder_after_terminate=False, delete_output_folder_after_terminate=False, shared_mode=False) backend = Backend(context) ens_builder = EnsembleBuilder( backend=backend, dataset_name=str(task_id), task_type=task_type, metric=metric, limit=np.inf, ensemble_size=ensemble_size, ensemble_nbest=max_keep_best, performance_range_threshold=performance_range_threshold, max_models_on_disc=None, seed=seed, shared_mode=False, precision=precision, max_iterations=1, read_at_most=1, memory_limit=memory_limit, random_state=1, sleep_duration=0) try: # iterate over all models, take construction time into account when creating new trajectory current_ensemble_timestamp = 0 skipped = 1 for midx, model_path in enumerate(n_models): tstamp = model_timestamps[model_path.split("/")[-1].replace( '.gz', '')] - overall_start_time if current_ensemble_timestamp > tstamp: # while this model was built, the ensemble script was not yet done skipped += 1 continue # Do one ensemble building step start = time.time() ens_builder.random_state = check_random_state(1) print("############## %d: Working on %s (skipped %d)" % (midx + 1, model_path, skipped - 1)) logging.basicConfig(level=logging.DEBUG) ens_builder.read_at_most = skipped valid_pred, test_pred = ens_builder.main(return_pred=True) last_dur = time.time() - start current_ensemble_timestamp = tstamp + last_dur if current_ensemble_timestamp >= vanilla_results["0"]["time_limit"]: print("############## Went over time %f > %f; Stop here" % (current_ensemble_timestamp, vanilla_results["0"]["time_limit"])) break # Reset, since we have just read model files skipped = 1 if test_pred is None: # Adding this model did not change the ensemble, no new prediction continue if task_type == BINARY_CLASSIFICATION: # Recreate nx2 array test_pred = np.concatenate([ 1 - test_pred.reshape([-1, 1]), test_pred.reshape([-1, 1]) ], axis=1) # Build trajectory entry score = 1 - balanced_accuracy(y_true=y_test, y_pred=test_pred) loss_trajectory.append((current_ensemble_timestamp, score)) print("############## Round %d took %g sec" % (midx, time.time() - start)) except: raise finally: tmp_dir.cleanup() # Store results result = dict() result[ensemble_size] = { 'task_id': task_id, 'time_limit': vanilla_results["0"]["time_limit"], 'loss': loss_trajectory[-1][1], 'configuration': { "n_models": n_models, "performance_range_threshold": performance_range_threshold, "ensemble_size": ensemble_size, "max_keep_best": max_keep_best, "seed": seed, "memory_limit": memory_limit, "precision": precision, }, 'n_models': len(n_models), 'trajectory': loss_trajectory, } with open(fl_name, 'wt') as fh: json.dump(result, fh, indent=4) print("Dumped to %s" % fl_name)
def test_fit_pSMAC(self): tmp = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC') output = os.path.join(self.test_dir, '..', '.out_estimator_fit_pSMAC') self._setUp(tmp) self._setUp(output) X_train, Y_train, X_test, Y_test = putil.get_dataset('breast_cancer') # test parallel Classifier to predict classes, not only indices Y_train += 1 Y_test += 1 automl = AutoSklearnClassifier( time_left_for_this_task=30, per_run_time_limit=5, output_folder=output, tmp_folder=tmp, shared_mode=True, seed=1, initial_configurations_via_metalearning=0, ensemble_size=0, ) automl.fit(X_train, Y_train) n_models_fit = len(automl.cv_results_['mean_test_score']) cv_results = automl.cv_results_['mean_test_score'] automl = AutoSklearnClassifier( time_left_for_this_task=30, per_run_time_limit=5, output_folder=output, tmp_folder=tmp, shared_mode=True, seed=2, initial_configurations_via_metalearning=0, ensemble_size=0, ) automl.fit(X_train, Y_train) n_models_fit_2 = len(automl.cv_results_['mean_test_score']) # Check that the results from the first run were actually read by the # second run self.assertGreater(n_models_fit_2, n_models_fit) for score in cv_results: self.assertIn( score, automl.cv_results_['mean_test_score'], msg=str((automl.cv_results_['mean_test_score'], cv_results)), ) # Create a 'dummy model' for the first run, which has an accuracy of # more than 99%; it should be in the final ensemble if the ensemble # building of the second AutoSklearn classifier works correct true_targets_ensemble_path = os.path.join(tmp, '.auto-sklearn', 'true_targets_ensemble.npy') with open(true_targets_ensemble_path, 'rb') as fh: true_targets_ensemble = np.load(fh, allow_pickle=True) true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0 true_targets_ensemble = true_targets_ensemble.astype(int) probas = np.zeros((len(true_targets_ensemble), 2), dtype=float) for i, value in enumerate(true_targets_ensemble): probas[i, value] = 1.0 dummy_predictions_path = os.path.join( tmp, '.auto-sklearn', 'predictions_ensemble', 'predictions_ensemble_0_999_0.0.npy', ) with open(dummy_predictions_path, 'wb') as fh: np.save(fh, probas) probas_test = np.zeros((len(Y_test), 2), dtype=float) for i, value in enumerate(Y_test): probas_test[i, value - 1] = 1.0 dummy = ArrayReturningDummyPredictor(probas_test) context = BackendContext(tmp, output, False, False, True) backend = Backend(context) model_path = backend.get_model_path(seed=0, idx=999, budget=0.0) backend.save_model(model=dummy, filepath=model_path) automl = AutoSklearnClassifier( time_left_for_this_task=30, per_run_time_limit=5, output_folder=output, tmp_folder=tmp, shared_mode=True, seed=3, initial_configurations_via_metalearning=0, ensemble_size=0, metric=accuracy, ) automl.fit_ensemble(Y_train, task=BINARY_CLASSIFICATION, precision='32', dataset_name='breast_cancer', ensemble_size=20, ensemble_nbest=50, ) predictions = automl.predict(X_test) score = sklearn.metrics.accuracy_score(Y_test, predictions) self.assertEqual(len(os.listdir(os.path.join(tmp, '.auto-sklearn', 'ensembles'))), 1) self.assertGreaterEqual(score, 0.90) self.assertEqual(automl._automl[0]._task, BINARY_CLASSIFICATION) models = automl._automl[0].models_ classifier_types = [type(c) for c in models.values()] self.assertIn(ArrayReturningDummyPredictor, classifier_types) del automl self._tearDown(tmp) self._tearDown(output)
def test_fit_pSMAC(self): tmp = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC') output = os.path.join(self.test_dir, '..', '.out_estimator_fit_pSMAC') self._setUp(tmp) self._setUp(output) X_train, Y_train, X_test, Y_test = putil.get_dataset('digits') # test parallel Classifier to predict classes, not only indexes Y_train += 1 Y_test += 1 automl = AutoSklearnClassifier( time_left_for_this_task=20, per_run_time_limit=5, output_folder=output, tmp_folder=tmp, shared_mode=True, seed=1, initial_configurations_via_metalearning=0, ensemble_size=0, ) automl.fit(X_train, Y_train) # Create a 'dummy model' for the first run, which has an accuracy of # more than 99%; it should be in the final ensemble if the ensemble # building of the second AutoSklearn classifier works correct true_targets_ensemble_path = os.path.join(tmp, '.auto-sklearn', 'true_targets_ensemble.npy') with open(true_targets_ensemble_path, 'rb') as fh: true_targets_ensemble = np.load(fh) true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0 true_targets_ensemble = true_targets_ensemble.astype(int) probas = np.zeros((len(true_targets_ensemble), 10), dtype=float) for i, value in enumerate(true_targets_ensemble): probas[i, value] = 1.0 dummy_predictions_path = os.path.join( tmp, '.auto-sklearn', 'predictions_ensemble', 'predictions_ensemble_1_00030.npy', ) with open(dummy_predictions_path, 'wb') as fh: np.save(fh, probas) probas_test = np.zeros((len(Y_test), 10), dtype=float) for i, value in enumerate(Y_test): probas_test[i, value - 1] = 1.0 dummy = ArrayReturningDummyPredictor(probas_test) context = BackendContext(tmp, output, False, False, True) backend = Backend(context) backend.save_model(dummy, 30, 1) automl = AutoSklearnClassifier( time_left_for_this_task=20, per_run_time_limit=5, output_folder=output, tmp_folder=tmp, shared_mode=True, seed=2, initial_configurations_via_metalearning=0, ensemble_size=0, ) automl.fit_ensemble(Y_train, task=MULTICLASS_CLASSIFICATION, metric=accuracy, precision='32', dataset_name='iris', ensemble_size=20, ensemble_nbest=50, ) predictions = automl.predict(X_test) score = sklearn.metrics.accuracy_score(Y_test, predictions) self.assertEqual(len(os.listdir(os.path.join(tmp, '.auto-sklearn', 'ensembles'))), 1) self.assertGreaterEqual(score, 0.90) self.assertEqual(automl._automl._task, MULTICLASS_CLASSIFICATION) models = automl._automl.models_ classifier_types = [type(c) for c in models.values()] self.assertIn(ArrayReturningDummyPredictor, classifier_types) del automl self._tearDown(tmp) self._tearDown(output)