def test_random_state_no_seed_different_params(): # Guarantees that specifying a random state relies in the successive # halving's having the same random state (and likewise for no random state) # This test is required because Hyperband wraps SHAs and the random state # needs to be passed correctly. values = scipy.stats.uniform(0, 1) max_iter = 9 brackets = _get_hyperband_params(max_iter) kwargs = {"value": values} h1 = HyperbandSearchCV(ConstantFunction(), kwargs, max_iter=max_iter) h2 = HyperbandSearchCV(ConstantFunction(), kwargs, max_iter=max_iter) h1._get_SHAs(brackets) h2._get_SHAs(brackets) assert h1._SHA_seed != h2._SHA_seed h1 = HyperbandSearchCV(ConstantFunction(), kwargs, max_iter=9, random_state=0) h2 = HyperbandSearchCV(ConstantFunction(), kwargs, max_iter=9, random_state=0) h1._get_SHAs(brackets) h2._get_SHAs(brackets) assert h1._SHA_seed == h2._SHA_seed
def _test_verbosity(c, s, a, b): X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} search = IncrementalSearchCV(model, params, max_iter=max_iter, verbose=verbose) yield search.fit(X, y) return search
def test_params_passed(): # This makes sure that the "SuccessiveHalvingSearchCV params" key in # Hyperband.metadata["brackets"] is correct and they can be instatiated. est = ConstantFunction(value=0.4) params = {"value": np.linspace(0, 1)} params = { "aggressiveness": 3.5, "max_iter": 253, "random_state": 42, "scoring": False, "test_size": 0.212, "tol": 0, } params["patience"] = (params["max_iter"] // params["aggressiveness"]) + 4 hyperband = HyperbandSearchCV(est, params, **params) for k, v in params.items(): assert getattr(hyperband, k) == v brackets = hyperband.metadata["brackets"] SHAs_params = [ bracket["SuccessiveHalvingSearchCV params"] for bracket in brackets ] for SHA_params in SHAs_params: for k, v in params.items(): if k == "random_state": continue assert SHA_params[k] == v seeds = [SHA_params["random_state"] for SHA_params in SHAs_params] assert len(set(seeds)) == len(seeds)
def test_successive_halving_params(c, s, a, b): # Makes sure when SHAs are fit with values from the "SuccessiveHalvingSearchCV # params" key, the number of models/calls stay the same as Hyperband. # This sanity check again makes sure parameters passed correctly # (similar to `test_params_passed`) X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} alg = HyperbandSearchCV(model, params, max_iter=27, random_state=42) kwargs = [ v["SuccessiveHalvingSearchCV params"] for v in alg.metadata["brackets"] ] SHAs = [SuccessiveHalvingSearchCV(model, params, **v) for v in kwargs] metadata = alg.metadata["brackets"] for k, (true_meta, SHA) in enumerate(zip(metadata, SHAs)): yield SHA.fit(X, y) n_models = len(SHA.model_history_) pf_calls = [ v[-1]["partial_fit_calls"] for v in SHA.model_history_.values() ] assert true_meta["n_models"] == n_models assert true_meta["partial_fit_calls"] == sum(pf_calls)
def _test_mirrors_paper(c, s, a, b): X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": np.random.rand(max_iter)} alg = HyperbandSearchCV( model, params, max_iter=max_iter, random_state=0, aggressiveness=aggressiveness, ) yield alg.fit(X, y) assert alg.metadata == alg.metadata_ assert isinstance(alg.metadata["brackets"], list) assert set(alg.metadata.keys()) == { "n_models", "partial_fit_calls", "brackets" } # Looping over alg.metadata["bracketes"] is okay because alg.metadata # == alg.metadata_ for bracket in alg.metadata["brackets"]: assert set(bracket.keys()) == { "n_models", "partial_fit_calls", "bracket", "SuccessiveHalvingSearchCV params", "decisions", } if aggressiveness == 3: assert alg.best_score_ == params["value"].max()
async def _test_verbosity(c, s, a, b): X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} search = Search(model, params, max_iter=max_iter, verbose=verbose) await search.fit(X, y) assert search.best_score_ > 0 # ensure search ran return search
def test_warns_scores_per_fit(c, s, a, b): X, y = make_classification(n_samples=100, n_features=5, chunks=10) params = {"value": np.random.RandomState(42).rand(1000)} model = ConstantFunction() search = IncrementalSearchCV(model, params, scores_per_fit=2) with pytest.warns(UserWarning, match="deprecated since Dask-ML v1.4.0"): yield search.fit(X, y)
def test_min_max_iter(c, s, a, b): # This test makes sure Hyperband works with max_iter=1. # Tests for max_iter < 1 are in test_incremental.py. values = scipy.stats.uniform(0, 1) X, y = make_classification(n_samples=10, n_features=4, chunks=10) max_iter = 1 h = HyperbandSearchCV(ConstantFunction(), {"value": values}, max_iter=max_iter) yield h.fit(X, y) assert h.best_score_ > 0
async def test_dataframe_inputs(c, s, a, b): X = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}) X = dd.from_pandas(X, npartitions=2) y = pd.Series([False, True, True]) y = dd.from_pandas(y, npartitions=2) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} alg = HyperbandSearchCV(model, params, max_iter=9, random_state=42) await alg.fit(X, y)
async def test_model_future(c, s, a, b): X, y = make_classification(n_samples=100, n_features=5, chunks=10) params = {"value": np.random.RandomState(42).rand(1000)} model = ConstantFunction() model_future = await c.scatter(model) search = IncrementalSearchCV(model_future, params, max_iter=10) await search.fit(X, y, classes=[0, 1]) assert search.history_ assert search.best_score_ > 0
def test_hyperband_patience(c, s, a, b): # Test to make sure that specifying patience=True results in less # computation X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} max_iter = 27 alg = HyperbandSearchCV(model, params, max_iter=max_iter, patience=True, random_state=0) yield alg.fit(X, y) alg_patience = max_iter // alg.aggressiveness actual_decisions = [b.pop("decisions") for b in alg.metadata_["brackets"]] paper_decisions = [b.pop("decisions") for b in alg.metadata["brackets"]] for paper_iter, actual_iter in zip(paper_decisions, actual_decisions): trimmed_paper_iter = {k for k in paper_iter if k <= alg_patience} # This makes sure that the algorithm is executed faithfully when # patience=True (and the proper decision points are preserved even if # other stop-on-plateau points are added) assert trimmed_paper_iter.issubset(set(actual_iter)) # This makes sure models aren't trained for too long assert all(x <= alg_patience + 1 for x in actual_iter) assert alg.metadata_["partial_fit_calls"] <= alg.metadata[ "partial_fit_calls"] assert alg.best_score_ >= 0.9 max_iter = 6 kwargs = dict(max_iter=max_iter, aggressiveness=2) alg = HyperbandSearchCV(model, params, patience=2, **kwargs) with pytest.warns(UserWarning, match="The goal of `patience`"): yield alg.fit(X, y) alg = HyperbandSearchCV(model, params, patience=2, tol=np.nan, **kwargs) yield alg.fit(X, y) assert pd.DataFrame(alg.history_).partial_fit_calls.max() == max_iter alg = HyperbandSearchCV(model, params, patience=2, tol=None, **kwargs) yield alg.fit(X, y) assert pd.DataFrame(alg.history_).partial_fit_calls.max() == max_iter alg = HyperbandSearchCV(model, params, patience=1, **kwargs) with pytest.raises(ValueError, match="always detect a plateau"): with warnings.catch_warnings(): warnings.simplefilter("ignore") yield alg.fit(X, y)
def test_cv_results_order_preserved(c, s, a, b): X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} alg = HyperbandSearchCV(model, params, max_iter=9, random_state=42) yield alg.fit(X, y) info = {k: v[-1] for k, v in alg.model_history_.items()} for _, row in pd.DataFrame(alg.cv_results_).iterrows(): model_info = info[row["model_id"]] assert row["bracket"] == model_info["bracket"] assert row["params"] == model_info["params"] assert np.allclose(row["test_score"], model_info["score"])
def test_verbosity_types(c, s, a, b): X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} for verbose in [-1.0, 1.2]: search = IncrementalSearchCV(model, params, verbose=verbose, max_iter=3) with pytest.raises(ValueError, match="0 <= verbose <= 1"): yield search.fit(X, y) for verbose in [0.0, 0, 1, 1.0, True, False]: search = IncrementalSearchCV(model, params, verbose=verbose, max_iter=3) yield search.fit(X, y)
def test_same_random_state_same_params(c, s, a, b): # This makes sure parameters are sampled correctly when random state is # specified. # This test makes sure random state is *correctly* passed to successive # halvings from Hyperband seed = 0 values = scipy.stats.uniform(0, 1) h = HyperbandSearchCV(ConstantFunction(), {"value": values}, random_state=seed, max_iter=9) # Make a class for passive random sampling passive = IncrementalSearchCV( ConstantFunction(), {"value": values}, random_state=seed, max_iter=2, n_initial_parameters=h.metadata["n_models"], ) X, y = make_classification(n_samples=10, n_features=4, chunks=10) yield h.fit(X, y) yield passive.fit(X, y) # Check to make sure the Hyperbands found the same params v_h = h.cv_results_["param_value"] # Check to make sure the random passive search had *some* of the same params v_passive = passive.cv_results_["param_value"] # Sanity checks to make sure all unique floats assert len(set(v_passive)) == len(v_passive) assert len(set(v_h)) == len(v_h) # Getting the `value`s that are the same for both searches same = set(v_passive).intersection(set(v_h)) passive_models = h.metadata["brackets"][0]["n_models"] assert len(same) == passive_models
async def test_warns_decay_rate(c, s, a, b): X, y = make_classification(n_samples=100, n_features=5, chunks=10) params = {"value": np.random.RandomState(42).rand(1000)} model = ConstantFunction() kwargs = dict(max_iter=5, n_initial_parameters=5) search = IncrementalSearchCV(model, params, **kwargs) match = r"deprecated since Dask-ML v1.4.0." with pytest.warns(FutureWarning, match=match): await search.fit(X, y) # Make sure the printed warning message works search = IncrementalSearchCV(model, params, decay_rate=None, **kwargs) await search.fit(X, y)
def test_warns_decay_rate_wanted(c, s, a, b): X, y = make_classification(n_samples=100, n_features=5, chunks=10) params = {"value": np.random.RandomState(42).rand(1000)} model = ConstantFunction() search = IncrementalSearchCV( model, params, max_iter=5, n_initial_parameters=5, decay_rate=1 ) match = "decay_rate is deprecated .* Use InverseDecaySearchCV" with pytest.warns(FutureWarning, match=match): yield search.fit(X, y) # Make sure old behavior is retained w/o warning search = InverseDecaySearchCV(model, params, decay_rate=1) yield search.fit(X, y)
def test_search_patience_infeasible_tol(c, s, a, b): X, y = make_classification(n_samples=100, n_features=5, chunks=(10, 5)) rng = check_random_state(42) params = {"value": rng.rand(1000)} model = ConstantFunction() max_iter = 10 score_increase = -10 search = IncrementalSearchCV( model, params, max_iter=max_iter, patience=3, tol=score_increase, ) yield search.fit(X, y, classes=[0, 1]) hist = pd.DataFrame(search.history_) assert hist.partial_fit_calls.max() == max_iter
def test_correct_params(c, s, a, b): # Makes sure that Hyperband has the correct parameters. # Implemented because Hyperband wraps SHA. Again, makes sure that parameters # are correctly passed to SHA (had a case where max_iter= flag not passed to # SuccessiveHalvingSearchCV but it should have been) est = ConstantFunction() X, y = make_classification(n_samples=10, n_features=4, chunks=10) params = {"value": np.linspace(0, 1)} search = HyperbandSearchCV(est, params, max_iter=9) base = { "estimator", "estimator__value", "estimator__sleep", "parameters", "max_iter", "test_size", "patience", "tol", "random_state", "scoring", "verbose", "prefix", } assert set(search.get_params().keys()) == base.union({"aggressiveness"}) meta = search.metadata SHAs_params = [ bracket["SuccessiveHalvingSearchCV params"] for bracket in meta["brackets"] ] SHA_params = base.union( { "n_initial_parameters", "n_initial_iter", "aggressiveness", "max_iter", "prefix", } ) - {"estimator__sleep", "estimator__value", "estimator", "parameters"} assert all(set(SHA) == SHA_params for SHA in SHAs_params) # this is testing to make sure that each SHA has the correct estimator yield search.fit(X, y) SHAs = search._SuccessiveHalvings_ assert all(search.estimator is SHA.estimator for SHA in SHAs.values()) assert all(search.parameters is SHA.parameters for SHA in SHAs.values())
def test_search_invalid_patience(c, s, a, b): X, y = make_classification(n_samples=100, n_features=5, chunks=10) params = {"value": np.random.RandomState(42).rand(1000)} model = ConstantFunction() search = IncrementalSearchCV(model, params, patience=1, max_iter=10) with pytest.raises(ValueError, match="patience >= 2"): yield search.fit(X, y, classes=[0, 1]) search = IncrementalSearchCV(model, params, patience=2.0, max_iter=10) with pytest.raises(ValueError, match="patience must be an integer"): yield search.fit(X, y, classes=[0, 1]) # Make sure this passes search = IncrementalSearchCV(model, params, patience=False, max_iter=10) yield search.fit(X, y, classes=[0, 1]) assert search.history_
def test_search_patience_infeasible_tol(c, s, a, b): X, y = make_classification(n_samples=100, n_features=5) params = {"value": np.random.RandomState(42).rand(1000)} model = ConstantFunction() search = SuccessiveHalvingSearchCV( model, params, patience=2, tol=np.nan, n_initial_parameters=20, n_initial_iter=4, max_iter=1000, ) yield search.fit(X, y, classes=[0, 1]) assert search.metadata_["partial_fit_calls"] == search.metadata[ "partial_fit_calls"] assert search.metadata_ == search.metadata
def test_history(c, s, a, b): # This test is required to make sure Hyperband wraps SHA successfully # Mostly, it's a test to make sure ordered by time # # There's also a test in test_incremental to make sure the history has # correct values/etc X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} alg = HyperbandSearchCV(model, params, max_iter=9, random_state=42) yield alg.fit(X, y) assert alg.cv_results_["model_id"].dtype == "<U11" assert all(isinstance(v, str) for v in alg.cv_results_["model_id"]) assert all("bracket=" in h["model_id"] for h in alg.history_) assert all("bracket" in h for h in alg.history_) # Hyperband does a custom ordering of times times = [v["elapsed_wall_time"] for v in alg.history_] assert (np.diff(times) >= 0).all() # Make sure results are ordered by partial fit calls for each model for model_hist in alg.model_history_.values(): calls = [h["partial_fit_calls"] for h in model_hist] assert (np.diff(calls) >= 1).all() or len(calls) == 1
def test_logs_dont_repeat(c, s, a, b): # This test is necessary to make sure the dask_ml.model_selection logger # isn't piped to stdout repeatedly. # # I developed this test to protect against this case: # getLogger("dask_ml.model_selection") is piped to stdout whenever a # bracket of Hyperband starts/each time SHA._fit is called X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} search = HyperbandSearchCV(model, params, max_iter=9, random_state=42) with captured_logger(logging.getLogger("dask_ml.model_selection")) as logs: yield search.fit(X, y) assert search.best_score_ > 0 # ensure search ran messages = logs.getvalue().splitlines() model_creation_msgs = [m for m in messages if "creating" in m] n_models = [m.split(" ")[-2] for m in model_creation_msgs] bracket_models = [b["n_models"] for b in search.metadata["brackets"]] assert len(bracket_models) == len(set(bracket_models)) # Make sure only one model creation message is printed per bracket # (all brackets have unique n_models as asserted above) assert len(n_models) == len(set(n_models))
def _test_basic(c, s, a, b): rng = da.random.RandomState(42) n, d = (50, 2) # create observations we know linear models can fit X = rng.normal(size=(n, d), chunks=n // 2) coef_star = rng.uniform(size=d, chunks=d) y = da.sign(X.dot(coef_star)) if array_type == "numpy": X, y = yield c.compute((X, y)) params = { "loss": ["hinge", "log", "modified_huber", "squared_hinge", "perceptron"], "average": [True, False], "learning_rate": ["constant", "invscaling", "optimal"], "eta0": np.logspace(-2, 0, num=1000), } model = SGDClassifier(tol=-np.inf, penalty="elasticnet", random_state=42, eta0=0.1) if library == "dask-ml": model = Incremental(model) params = {"estimator__" + k: v for k, v in params.items()} elif library == "ConstantFunction": model = ConstantFunction() params = {"value": np.linspace(0, 1, num=1000)} search = HyperbandSearchCV(model, params, max_iter=max_iter, random_state=42) classes = c.compute(da.unique(y)) yield search.fit(X, y, classes=classes) if library == "dask-ml": X, y = yield c.compute((X, y)) score = search.best_estimator_.score(X, y) assert score == search.score(X, y) assert 0 <= score <= 1 if library == "ConstantFunction": assert score == search.best_score_ else: # These are not equal because IncrementalSearchCV uses a train/test # split and we're testing on the entire train dataset, not only the # validation/test set. assert abs(score - search.best_score_) < 0.1 assert type(search.best_estimator_) == type(model) assert isinstance(search.best_params_, dict) num_fit_models = len(set(search.cv_results_["model_id"])) num_pf_calls = sum([ v[-1]["partial_fit_calls"] for v in search.model_history_.values() ]) models = {9: 17, 15: 17, 20: 17, 27: 49, 30: 49, 81: 143} pf_calls = {9: 69, 15: 101, 20: 144, 27: 357, 30: 379, 81: 1581} assert num_fit_models == models[max_iter] assert num_pf_calls == pf_calls[max_iter] best_idx = search.best_index_ if isinstance(model, ConstantFunction): assert search.cv_results_["test_score"][best_idx] == max( search.cv_results_["test_score"]) model_ids = {h["model_id"] for h in search.history_} if math.log(max_iter, 3) % 1.0 == 0: # log(max_iter, 3) % 1.0 == 0 is the good case when max_iter is a # power of search.aggressiveness # In this case, assert that more models are tried then the max_iter assert len(model_ids) > max_iter else: # Otherwise, give some padding "almost as many estimators are tried # as max_iter". 3 is a fudge number chosen to be the minimum; when # max_iter=20, len(model_ids) == 17. assert len(model_ids) + 3 >= max_iter assert all("bracket" in id_ for id_ in model_ids)
async def test_basic(c, s, a, b): def _additional_calls(info): pf_calls = {k: v[-1]["partial_fit_calls"] for k, v in info.items()} ret = {k: int(calls < 10) for k, calls in pf_calls.items()} if len(ret) == 1: return {list(ret)[0]: 0} # Don't train one model (but keep model 0) some_keys = set(ret.keys()) - {0} key_to_drop = random.choice(list(some_keys)) return {k: v for k, v in ret.items() if k != key_to_drop} X, y = make_classification(n_samples=1000, n_features=5, chunks=100) model = ConstantFunction() params = {"value": uniform(0, 1)} X_test, y_test = X[:100], y[:100] X_train = X[100:] y_train = y[100:] n_parameters = 5 param_list = list(ParameterSampler(params, n_parameters)) info, models, history, best = await fit( model, param_list, X_train, y_train, X_test, y_test, _additional_calls, fit_params={"classes": [0, 1]}, ) # Ensure that we touched all data keys = {t[0] for t in s.transition_log} L = [str(k) in keys for kk in X_train.__dask_keys__() for k in kk] assert all(L) for model in models.values(): assert isinstance(model, Future) model2 = await model assert isinstance(model2, ConstantFunction) XX_test = await c.compute(X_test) yy_test = await c.compute(y_test) model = await models[0] assert model.score(XX_test, yy_test) == info[0][-1]["score"] # `<` not `==` because we randomly dropped one model every iteration assert len(history) < n_parameters * 10 for h in history: assert { "partial_fit_time", "score_time", "score", "model_id", "params", "partial_fit_calls", }.issubset(set(h.keys())) groups = toolz.groupby("partial_fit_calls", history) assert len(groups[1]) > len(groups[2]) > len(groups[3]) > len(groups[max(groups)]) assert max(groups) == n_parameters keys = list(models.keys()) for key in keys: del models[key] while c.futures or s.tasks: # Make sure cleans up cleanly after running await asyncio.sleep(0.1) # smoke test for ndarray X_test and y_test X_test = await c.compute(X_test) y_test = await c.compute(y_test) info, models, history, best = await fit( model, param_list, X_train, y_train, X_test, y_test, _additional_calls, fit_params={"classes": [0, 1]}, ) assert True # smoke test to make sure reached
def test_history(c, s, a, b): X, y = make_classification(n_samples=10, n_features=4, chunks=10) model = ConstantFunction() params = {"value": scipy.stats.uniform(0, 1)} alg = IncrementalSearchCV(model, params, max_iter=9, random_state=42) yield alg.fit(X, y) gt_zero = lambda x: x >= 0 gt_one = lambda x: x >= 1 key_types_and_checks = [ ("mean_partial_fit_time", float, gt_zero), ("mean_score_time", float, gt_zero), ("std_partial_fit_time", float, gt_zero), ("std_score_time", float, gt_zero), ("test_score", float, gt_zero), ("rank_test_score", int, gt_one), ("model_id", int, None), ("partial_fit_calls", int, gt_zero), ("params", dict, lambda d: set(d.keys()) == {"value"}), ("param_value", float, gt_zero), ] assert set(alg.cv_results_) == {v[0] for v in key_types_and_checks} for column, dtype, condition in key_types_and_checks: if dtype: assert alg.cv_results_[column].dtype == dtype if condition: assert all(condition(x) for x in alg.cv_results_[column]) alg.best_estimator_.fit(X, y) alg.best_estimator_.score(X, y) alg.score(X, y) # Test types/format of all parameters we set after fitting assert isinstance(alg.best_index_, int) assert isinstance(alg.best_estimator_, ConstantFunction) assert isinstance(alg.best_score_, float) assert isinstance(alg.best_params_, dict) assert isinstance(alg.history_, list) assert all(isinstance(h, dict) for h in alg.history_) assert isinstance(alg.model_history_, dict) assert all(vi in alg.history_ for v in alg.model_history_.values() for vi in v) assert all(isinstance(v, np.ndarray) for v in alg.cv_results_.values()) assert isinstance(alg.multimetric_, bool) keys = { "score", "score_time", "partial_fit_calls", "partial_fit_time", "model_id", "elapsed_wall_time", "params", } assert all(set(h.keys()) == keys for h in alg.history_) times = [v["elapsed_wall_time"] for v in alg.history_] assert (np.diff(times) >= 0).all() # Test to make sure history_ ordered with wall time assert (np.diff([v["elapsed_wall_time"] for v in alg.history_]) >= 0).all() for model_hist in alg.model_history_.values(): calls = [h["partial_fit_calls"] for h in model_hist] assert (np.diff(calls) >= 1).all() or len(calls) == 1