def test_hyperband_patience(c, s, a, b): # Test to make sure that specifying patience=True results in less # computation X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} max_iter = 27 alg = HyperbandSearchCV(model, params, max_iter=max_iter, patience=True, random_state=0) yield alg.fit(X, y) alg_patience = max_iter // alg.aggressiveness actual_decisions = [b.pop("decisions") for b in alg.metadata_["brackets"]] paper_decisions = [b.pop("decisions") for b in alg.metadata["brackets"]] for paper_iter, actual_iter in zip(paper_decisions, actual_decisions): trimmed_paper_iter = {k for k in paper_iter if k <= alg_patience} # This makes sure that the algorithm is executed faithfully when # patience=True (and the proper decision points are preserved even if # other stop-on-plateau points are added) assert trimmed_paper_iter.issubset(set(actual_iter)) # This makes sure models aren't trained for too long assert all(x <= alg_patience + 1 for x in actual_iter) assert alg.metadata_["partial_fit_calls"] <= alg.metadata[ "partial_fit_calls"] assert alg.best_score_ >= 0.9 max_iter = 6 kwargs = dict(max_iter=max_iter, aggressiveness=2) alg = HyperbandSearchCV(model, params, patience=2, **kwargs) with pytest.warns(UserWarning, match="The goal of `patience`"): yield alg.fit(X, y) alg = HyperbandSearchCV(model, params, patience=2, tol=np.nan, **kwargs) yield alg.fit(X, y) assert pd.DataFrame(alg.history_).partial_fit_calls.max() == max_iter alg = HyperbandSearchCV(model, params, patience=2, tol=None, **kwargs) yield alg.fit(X, y) assert pd.DataFrame(alg.history_).partial_fit_calls.max() == max_iter alg = HyperbandSearchCV(model, params, patience=1, **kwargs) with pytest.raises(ValueError, match="always detect a plateau"): with warnings.catch_warnings(): warnings.simplefilter("ignore") yield alg.fit(X, y)
def _test_mirrors_paper(c, s, a, b): X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": np.random.rand(max_iter)} alg = HyperbandSearchCV( model, params, max_iter=max_iter, random_state=0, aggressiveness=aggressiveness, ) yield alg.fit(X, y) assert alg.metadata == alg.metadata_ assert isinstance(alg.metadata["brackets"], list) assert set(alg.metadata.keys()) == { "n_models", "partial_fit_calls", "brackets" } # Looping over alg.metadata["bracketes"] is okay because alg.metadata # == alg.metadata_ for bracket in alg.metadata["brackets"]: assert set(bracket.keys()) == { "n_models", "partial_fit_calls", "bracket", "SuccessiveHalvingSearchCV params", "decisions", } if aggressiveness == 3: assert alg.best_score_ == params["value"].max()
def tune_dask(clf, params, X_train, y_train, X_test, y_test, n_params=-1, max_epochs=-1, n_jobs=4): clf = clone(clf).set_params(prefix="dask") chunk_size = len(X_train) * max_epochs // n_params max_iter = n_params print(f"max_iter, chunk_size = {max_iter}, {chunk_size}") print(f"n_params = {n_params}") X_train = da.from_array(X_train).rechunk(chunks=(chunk_size, -1)) y_train = da.from_array(y_train).rechunk(chunks=chunk_size) print(y_train.chunks) search = HyperbandSearchCV( clf, params, max_iter=max_iter, random_state=42, test_size=0.2, aggressiveness=4, ) meta = search.metadata hmeta = {k: meta[k] for k in ["n_models", "partial_fit_calls"]} start = time() search.fit(X_train, y_train) fit_time = time() - start data = { "library": "dask", "best_score": search.best_score_, "best_params": search.best_params_, "fit_time": fit_time, "start_time": start, "n_params": n_params, "n_jobs": n_jobs, "max_epochs": max_epochs, **hmeta, } return search, data
def test_min_max_iter(c, s, a, b): # This test makes sure Hyperband works with max_iter=1. # Tests for max_iter < 1 are in test_incremental.py. values = scipy.stats.uniform(0, 1) X, y = make_classification(n_samples=10, n_features=4, chunks=10) max_iter = 1 h = HyperbandSearchCV(ConstantFunction(), {"value": values}, max_iter=max_iter) yield h.fit(X, y) assert h.best_score_ > 0
def test_cv_results_order_preserved(c, s, a, b): X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} alg = HyperbandSearchCV(model, params, max_iter=9, random_state=42) yield alg.fit(X, y) info = {k: v[-1] for k, v in alg.model_history_.items()} for _, row in pd.DataFrame(alg.cv_results_).iterrows(): model_info = info[row["model_id"]] assert row["bracket"] == model_info["bracket"] assert row["params"] == model_info["params"] assert np.allclose(row["test_score"], model_info["score"])
def test_correct_params(c, s, a, b): # Makes sure that Hyperband has the correct parameters. # Implemented because Hyperband wraps SHA. Again, makes sure that parameters # are correctly passed to SHA (had a case where max_iter= flag not passed to # SuccessiveHalvingSearchCV but it should have been) est = ConstantFunction() X, y = make_classification(n_samples=10, n_features=4, chunks=10) params = {"value": np.linspace(0, 1)} search = HyperbandSearchCV(est, params, max_iter=9) base = { "estimator", "estimator__value", "estimator__sleep", "parameters", "max_iter", "test_size", "patience", "tol", "random_state", "scoring", "verbose", "prefix", } assert set(search.get_params().keys()) == base.union({"aggressiveness"}) meta = search.metadata SHAs_params = [ bracket["SuccessiveHalvingSearchCV params"] for bracket in meta["brackets"] ] SHA_params = base.union( { "n_initial_parameters", "n_initial_iter", "aggressiveness", "max_iter", "prefix", } ) - {"estimator__sleep", "estimator__value", "estimator", "parameters"} assert all(set(SHA) == SHA_params for SHA in SHAs_params) # this is testing to make sure that each SHA has the correct estimator yield search.fit(X, y) SHAs = search._SuccessiveHalvings_ assert all(search.estimator is SHA.estimator for SHA in SHAs.values()) assert all(search.parameters is SHA.parameters for SHA in SHAs.values())
def test_same_random_state_same_params(c, s, a, b): # This makes sure parameters are sampled correctly when random state is # specified. # This test makes sure random state is *correctly* passed to successive # halvings from Hyperband seed = 0 values = scipy.stats.uniform(0, 1) h = HyperbandSearchCV(ConstantFunction(), {"value": values}, random_state=seed, max_iter=9) # Make a class for passive random sampling passive = IncrementalSearchCV( ConstantFunction(), {"value": values}, random_state=seed, max_iter=2, n_initial_parameters=h.metadata["n_models"], ) X, y = make_classification(n_samples=10, n_features=4, chunks=10) yield h.fit(X, y) yield passive.fit(X, y) # Check to make sure the Hyperbands found the same params v_h = h.cv_results_["param_value"] # Check to make sure the random passive search had *some* of the same params v_passive = passive.cv_results_["param_value"] # Sanity checks to make sure all unique floats assert len(set(v_passive)) == len(v_passive) assert len(set(v_h)) == len(v_h) # Getting the `value`s that are the same for both searches same = set(v_passive).intersection(set(v_h)) passive_models = h.metadata["brackets"][0]["n_models"] assert len(same) == passive_models
def test_history(c, s, a, b): # This test is required to make sure Hyperband wraps SHA successfully # Mostly, it's a test to make sure ordered by time # # There's also a test in test_incremental to make sure the history has # correct values/etc X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} alg = HyperbandSearchCV(model, params, max_iter=9, random_state=42) yield alg.fit(X, y) assert alg.cv_results_["model_id"].dtype == "<U11" assert all(isinstance(v, str) for v in alg.cv_results_["model_id"]) assert all("bracket=" in h["model_id"] for h in alg.history_) assert all("bracket" in h for h in alg.history_) # Hyperband does a custom ordering of times times = [v["elapsed_wall_time"] for v in alg.history_] assert (np.diff(times) >= 0).all() # Make sure results are ordered by partial fit calls for each model for model_hist in alg.model_history_.values(): calls = [h["partial_fit_calls"] for h in model_hist] assert (np.diff(calls) >= 1).all() or len(calls) == 1
def test_logs_dont_repeat(c, s, a, b): # This test is necessary to make sure the dask_ml.model_selection logger # isn't piped to stdout repeatedly. # # I developed this test to protect against this case: # getLogger("dask_ml.model_selection") is piped to stdout whenever a # bracket of Hyperband starts/each time SHA._fit is called X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} search = HyperbandSearchCV(model, params, max_iter=9, random_state=42) with captured_logger(logging.getLogger("dask_ml.model_selection")) as logs: yield search.fit(X, y) assert search.best_score_ > 0 # ensure search ran messages = logs.getvalue().splitlines() model_creation_msgs = [m for m in messages if "creating" in m] n_models = [m.split(" ")[-2] for m in model_creation_msgs] bracket_models = [b["n_models"] for b in search.metadata["brackets"]] assert len(bracket_models) == len(set(bracket_models)) # Make sure only one model creation message is printed per bracket # (all brackets have unique n_models as asserted above) assert len(n_models) == len(set(n_models))
def _test_basic(c, s, a, b): rng = da.random.RandomState(42) n, d = (50, 2) # create observations we know linear models can fit X = rng.normal(size=(n, d), chunks=n // 2) coef_star = rng.uniform(size=d, chunks=d) y = da.sign(X.dot(coef_star)) if array_type == "numpy": X, y = yield c.compute((X, y)) params = { "loss": ["hinge", "log", "modified_huber", "squared_hinge", "perceptron"], "average": [True, False], "learning_rate": ["constant", "invscaling", "optimal"], "eta0": np.logspace(-2, 0, num=1000), } model = SGDClassifier(tol=-np.inf, penalty="elasticnet", random_state=42, eta0=0.1) if library == "dask-ml": model = Incremental(model) params = {"estimator__" + k: v for k, v in params.items()} elif library == "ConstantFunction": model = ConstantFunction() params = {"value": np.linspace(0, 1, num=1000)} search = HyperbandSearchCV(model, params, max_iter=max_iter, random_state=42) classes = c.compute(da.unique(y)) yield search.fit(X, y, classes=classes) if library == "dask-ml": X, y = yield c.compute((X, y)) score = search.best_estimator_.score(X, y) assert score == search.score(X, y) assert 0 <= score <= 1 if library == "ConstantFunction": assert score == search.best_score_ else: # These are not equal because IncrementalSearchCV uses a train/test # split and we're testing on the entire train dataset, not only the # validation/test set. assert abs(score - search.best_score_) < 0.1 assert type(search.best_estimator_) == type(model) assert isinstance(search.best_params_, dict) num_fit_models = len(set(search.cv_results_["model_id"])) num_pf_calls = sum([ v[-1]["partial_fit_calls"] for v in search.model_history_.values() ]) models = {9: 17, 15: 17, 20: 17, 27: 49, 30: 49, 81: 143} pf_calls = {9: 69, 15: 101, 20: 144, 27: 357, 30: 379, 81: 1581} assert num_fit_models == models[max_iter] assert num_pf_calls == pf_calls[max_iter] best_idx = search.best_index_ if isinstance(model, ConstantFunction): assert search.cv_results_["test_score"][best_idx] == max( search.cv_results_["test_score"]) model_ids = {h["model_id"] for h in search.history_} if math.log(max_iter, 3) % 1.0 == 0: # log(max_iter, 3) % 1.0 == 0 is the good case when max_iter is a # power of search.aggressiveness # In this case, assert that more models are tried then the max_iter assert len(model_ids) > max_iter else: # Otherwise, give some padding "almost as many estimators are tried # as max_iter". 3 is a fudge number chosen to be the minimum; when # max_iter=20, len(model_ids) == 17. assert len(model_ids) + 3 >= max_iter assert all("bracket" in id_ for id_ in model_ids)
y_train = y_train.to_dask_array(lengths=True) # In[15]: # rechunk the array for the partial fit X_train = X_train.rechunk((chunks, -1)) y_train = y_train.rechunk(chunks) # In[16]: print ("Starting hyperband search") ml_hyperband.fit(X=X_train,y=y_train) # In[17]: print (f"best train score :{ml_hyperband.best_score_}") print (f"best estimator : \n{ml_hyperband.best_estimator_}") print (f"Score on test set : {ml_hyperband.score(X_test, y_test)}") # In[18]: def evaluate(model, test_features, test_labels): predictions = model.predict(test_features.compute())
"module__activation": [ "relu", "elu", ], "batch_size": [32, 64], "optimizer__lr": loguniform(1e-4, 1e-3), "optimizer__weight_decay": loguniform(1e-6, 1e-3), "optimizer__momentum": uniform(0, 1), "optimizer__nesterov": [True], } from dask_ml.model_selection import HyperbandSearchCV search = HyperbandSearchCV(model, params, random_state=2, verbose=True, max_iter=9) y_train2 = y_train.reshape(-1, 1).persist() search.fit(X_train, y_train2) print(search.best_score_) print(search.best_params_) print(search.best_estimator_) from dask_ml.wrappers import ParallelPostFit deployed_model = ParallelPostFit(search.best_estimator_) deployed_model.score(X_test, y_test)