def test_shared_memory_1(): c_time = time.time() hyper = Hyperactive(n_processes=1) hyper.add_search( model, search_space, n_iter=100, n_jobs=1, ) hyper.run() d_time_1 = time.time() - c_time c_time = time.time() hyper = Hyperactive(n_processes=1) hyper.add_search( model, search_space, n_iter=100, n_jobs=4, ) hyper.run() d_time_2 = time.time() - c_time print("\n d_time_1 \n", d_time_1) print("\n d_time_2 \n", d_time_2) d_time_2 = d_time_2 / 2 assert d_time_1 / d_time_2 > 1.1
def test_memory_timeSave_1(): data = load_breast_cancer() X, y = data.data, data.target def objective_function(opt): time.sleep(0.001) return 1 search_space = { "x1": list(np.arange(1, 30)), } results = pd.DataFrame(np.arange(1, 30), columns=["x1"]) results["score"] = 0 c_time1 = time.time() hyper = Hyperactive() hyper.add_search( objective_function, search_space, n_iter=1000, memory_warm_start=results, ) hyper.run() diff_time1 = time.time() - c_time1 print("diff_time1", diff_time1) assert diff_time1 < 1
def test_data_collector_2(): path = "./search_data.csv" if os.path.isfile(path): os.remove(path) data_c = DataCollector(path, drop_duplicates=False) def objective_function(opt): score = -opt["x1"] * opt["x1"] para_dict = { "x1": opt["x1"], "score": score, } data_c.append(para_dict) return score search_space = { "x1": np.arange(-100, 101, 1), } hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=50, memory=False) hyper.run() search_data = hyper.results(objective_function)[["x1", "score"]] search_data_ = data_c.load() assert df_equal(search_data, search_data_)
def test_shared_memory_warm_start_0(): c_time = time.time() hyper = Hyperactive(n_processes=1) hyper.add_search( model, search_space, n_iter=100, n_jobs=1, ) hyper.run() d_time_1 = time.time() - c_time search_data0 = hyper.results(model) c_time = time.time() hyper = Hyperactive(n_processes=1) hyper.add_search( model, search_space, n_iter=100, n_jobs=4, memory_warm_start=search_data0, ) hyper.run() d_time_2 = time.time() - c_time print("\n d_time_1 \n", d_time_1) print("\n d_time_2 \n", d_time_2) d_time_2 = d_time_2 / 2 assert d_time_2 * 1.4 - d_time_1 < 0
def test_data_collector_3(): if os.path.isfile(path): os.remove(path) search_space = { "x1": np.arange(-100, 101, 0.001), } hyper = Hyperactive() hyper.add_search( objective_function_3, search_space, n_iter=5, n_jobs=2, memory=False, initialize={"random": 1}, ) hyper.run() search_data = hyper.results(objective_function_3) search_data_ = data_c.load() print("\n search_data \n", search_data) print("\n search_data_ \n", search_data_) assert df_equal(search_data, search_data_)
def test_search_space_3(): def func1(): pass def func2(): pass def func3(): pass def objective_function(opt): score = -opt["x1"] * opt["x1"] return score search_space = { "x1": list(range(0, 100, 1)), "func1": [func1, func2, func3], } hyper = Hyperactive() hyper.add_search( objective_function, search_space, n_iter=15, ) hyper.run() assert isinstance(hyper.results(objective_function), pd.DataFrame) assert (hyper.best_para(objective_function)["func1"] in search_space["func1"])
def test_search_space_4(): class class1: pass class class2: pass class class3: pass def objective_function(opt): score = -opt["x1"] * opt["x1"] return score search_space = { "x1": list(range(0, 100, 1)), "class1": [class1, class2, class3], } hyper = Hyperactive() hyper.add_search( objective_function, search_space, n_iter=15, ) hyper.run() assert isinstance(hyper.results(objective_function), pd.DataFrame) assert (hyper.best_para(objective_function)["class1"] in search_space["class1"])
def test_attributes_results_6(): def objective_function(optimizer): score = (-optimizer.suggested_params["x1"] * optimizer.suggested_params["x1"]) return score search_space = { "x1": np.arange(0, 10, 1), } hyper = Hyperactive() hyper.add_search( objective_function, search_space, n_iter=20, initialize={"random": 1}, memory=False, ) hyper.run() x1_results = list(hyper.results(objective_function)["x1"].values) print("\n x1_results \n", x1_results) assert len(set(x1_results)) < len(x1_results)
def test_filter_data_0(): search_id1 = "test_model1" search_id2 = "test_model2" search_id3 = "test_model3" search_ids = [search_id1, search_id2, search_id3] def objective_function(opt): score = -opt["x1"] * opt["x1"] return score search_space = { "x1": np.arange(-100, 101, 1), } hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=200) hyper.run() search_data = hyper.results(objective_function) indices = list(search_space.keys()) + ["score"] filter_dict = { "parameter": indices, "lower bound": "---", "upper bound": "---", } filter_df = pd.DataFrame(filter_dict) threshold = -1000 filter_df["lower bound"].iloc[1] = threshold board = StreamlitBackend(search_ids) progress_data = board.filter_data(search_data, filter_df) assert not np.all(search_data["score"].values >= threshold) assert np.all(progress_data["score"].values >= threshold)
def test_ltm_10(): path = None model_name = "test_ltm_0" def objective_function(opt): score = -opt["x1"] * opt["x1"] return score search_space = { "x1": list(range(0, 3, 1)), } memory = LongTermMemory(model_name, path=path) hyper = Hyperactive() hyper.add_search( objective_function, search_space, n_iter=25, long_term_memory=memory ) hyper.run() hyper = Hyperactive() hyper.add_search( objective_function, search_space, n_iter=25, long_term_memory=memory ) hyper.run() memory.remove_model_data()
def test_issue_29(): data = load_boston() X, y = data.data, data.target def model(para): gbr = GradientBoostingRegressor( n_estimators=para["n_estimators"], max_depth=para["max_depth"], min_samples_split=para["min_samples_split"], ) scores = cross_val_score(gbr, X, y, cv=3) print( "Iteration:", para.optimizer.nth_iter, " Best score", para.optimizer.best_score, ) return scores.mean() search_space = { "n_estimators": list(range(10, 150, 5)), "max_depth": list(range(2, 12)), "min_samples_split": list(range(2, 22)), } hyper = Hyperactive() hyper.add_search(model, search_space, n_iter=20) hyper.run()
def test_streamlit_backend_2(): search_id1 = "test_model1" search_id2 = "test_model2" search_id3 = "test_model3" search_ids = [search_id1, search_id2, search_id3] board = StreamlitBackend(search_ids) def objective_function(opt): score = -opt["x1"] * opt["x1"] return score search_space = { "x1": np.arange(-100, 101, 1), } hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=200) hyper.run() search_data = hyper.results(objective_function) search_data["nth_iter"] = 0 search_data["score_best"] = 0 search_data["nth_process"] = 0 search_data["best"] = 0 plotly_fig = board.plotly(search_data, search_id1) assert plotly_fig is not None
def test_ltm_0(objective_function, search_space, path): (search_space, compare) = search_space print("\n objective_function \n", objective_function) print("\n search_space \n", search_space) print("\n compare \n", compare) print("\n path \n", path) model_name = str(objective_function.__name__) hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=10, initialize={"random": 1}) hyper.run() results1 = hyper.results(objective_function) memory = LongTermMemory(model_name, path=path) memory.save(results1, objective_function) results2 = memory.load() print("\n results1 \n", results1) print("\n results2 \n", results2) memory.remove_model_data() compare(results1, results2)
def test_early_stop_3(): def objective_function(para): score = -para["x1"] * para["x1"] return score search_space = { "x1": np.arange(0, 100, 0.1), } n_iter_no_change = 5 early_stopping = { "n_iter_no_change": n_iter_no_change, } hyper = Hyperactive() hyper.add_search( objective_function, search_space, n_iter=100000, initialize={"warm_start": [{ "x1": 0 }]}, early_stopping=early_stopping, ) hyper.run() search_data = hyper.results(objective_function) n_performed_iter = len(search_data) print("\n n_performed_iter \n", n_performed_iter) print("\n n_iter_no_change \n", n_iter_no_change) assert n_performed_iter == (n_iter_no_change + 1)
def test_trafo_1(random_state, search_space, memory_warm_start): search_space = search_space memory_warm_start = memory_warm_start c_time_0 = time.time() hyper = Hyperactive() hyper.add_search( objective_function_1, search_space, n_iter=10, random_state=random_state, initialize={"random": 1}, ) hyper.run() d_time_0 = time.time() - c_time_0 c_time_1 = time.time() hyper = Hyperactive() hyper.add_search( objective_function_1, search_space, n_iter=10, random_state=random_state, initialize={"random": 1}, memory_warm_start=memory_warm_start, ) hyper.run() d_time_1 = time.time() - c_time_1 assert d_time_1 < d_time_0 * 0.5
def test_attributes_results_1(): hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=100) hyper.run() assert set(search_space.keys()) < set( hyper.results(objective_function).columns)
def test_max_score_1(): def objective_function(para): score = -para["x1"] * para["x1"] time.sleep(0.01) return score search_space = { "x1": np.arange(0, 100, 0.1), } max_score = -9999 c_time = time.time() hyper = Hyperactive() hyper.add_search( objective_function, search_space, n_iter=100000, initialize={"warm_start": [{ "x1": 99 }]}, max_score=max_score, ) hyper.run() diff_time = time.time() - c_time print("\n Results head \n", hyper.results(objective_function).head()) print("\n Results tail \n", hyper.results(objective_function).tail()) print("\nN iter:", len(hyper.results(objective_function))) assert diff_time < 1
def test_memory_warm_start_1(search_space): c_time = time.time() hyper0 = Hyperactive() hyper0.add_search(objective_function, search_space, n_iter=2000, memory=False) hyper0.run() d_time_1 = time.time() - c_time search_data0 = hyper0.results(objective_function) c_time = time.time() hyper1 = Hyperactive() hyper1.add_search(objective_function, search_space, n_iter=2000, memory_warm_start=search_data0) hyper1.run() d_time_2 = time.time() - c_time print("\n d_time_1 ", d_time_1) print("\n d_time_2 ", d_time_2) assert d_time_1 * 0.5 > d_time_2
def test_random_state_2(): hyper0 = Hyperactive() hyper0.add_search( objective_function, search_space, n_iter=10, initialize={"random": 1}, random_state=1, ) hyper0.run() hyper1 = Hyperactive() hyper1.add_search( objective_function, search_space, n_iter=10, initialize={"random": 1}, random_state=10, ) hyper1.run() best_score0 = hyper0.best_score(objective_function) best_score1 = hyper1.best_score(objective_function) assert abs(best_score0 - best_score1) > err
def test_n_processes_5(): hyper = Hyperactive(n_processes=1) hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=4) hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=4) hyper.run() assert len(hyper.results_list) == 8
def test_memory_timeSave_0(): data = load_breast_cancer() X, y = data.data, data.target def objective_function(opt): dtc = DecisionTreeClassifier( min_samples_split=opt["min_samples_split"]) scores = cross_val_score(dtc, X, y, cv=5) return scores.mean() search_space = { "min_samples_split": np.arange(2, 20), } c_time1 = time.time() hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=100) hyper.run() diff_time1 = time.time() - c_time1 c_time2 = time.time() hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=100, memory=False) hyper.run() diff_time2 = time.time() - c_time2 assert diff_time1 < diff_time2 * 0.8
def test_memory_timeSave_1(): data = load_breast_cancer() X, y = data.data, data.target def objective_function(opt): dtc = DecisionTreeClassifier(max_depth=opt["max_depth"]) scores = cross_val_score(dtc, X, y, cv=5) return scores.mean() search_space = { "max_depth": list(np.arange(1, 101)), } results = pd.DataFrame(np.arange(1, 101), columns=["max_depth"]) results["score"] = 0 c_time1 = time.time() hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=300, memory_warm_start=results) hyper.run() diff_time1 = time.time() - c_time1 assert diff_time1 < 1
def test_memory_warm_start_manual(): data = load_breast_cancer() X, y = data.data, data.target def objective_function(opt): dtc = GradientBoostingClassifier(n_estimators=opt["n_estimators"], ) scores = cross_val_score(dtc, X, y, cv=5) return scores.mean() search_space = { "n_estimators": list(np.arange(500, 502)), } c_time_1 = time.time() hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=1) hyper.run() diff_time_1 = time.time() - c_time_1 memory_warm_start = pd.DataFrame([[500, 0.9], [501, 0.91]], columns=["n_estimators", "score"]) c_time = time.time() hyper0 = Hyperactive() hyper0.add_search( objective_function, search_space, n_iter=10, memory_warm_start=memory_warm_start, ) hyper0.run() diff_time = time.time() - c_time assert diff_time_1 > diff_time * 0.3
def test_attributes_result_errors_2(): with pytest.raises(ValueError): hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=15) hyper.run() hyper.results(objective_function1)
def test_max_score_0(): def objective_function(para): score = -para["x1"] * para["x1"] return score search_space = { "x1": np.arange(0, 100, 0.1), } max_score = -9999 opt = HillClimbingOptimizer( epsilon=0.01, rand_rest_p=0, ) hyper = Hyperactive() hyper.add_search( objective_function, search_space, optimizer=opt, n_iter=100000, initialize={"warm_start": [{ "x1": 99 }]}, max_score=max_score, ) hyper.run() print("\n Results head \n", hyper.results(objective_function).head()) print("\n Results tail \n", hyper.results(objective_function).tail()) print("\nN iter:", len(hyper.results(objective_function))) assert -100 > hyper.best_score(objective_function) > max_score
def test_early_stop_6(): def objective_function(para): return para["x1"] search_space = { "x1": np.arange(0, 100, 0.01), } n_iter_no_change = 5 early_stopping = { "n_iter_no_change": 5, "tol_abs": None, "tol_rel": 10, } start1 = {"x1": 1} start2 = {"x1": 1.1} start3 = {"x1": 1.22} start4 = {"x1": 1.35} start5 = {"x1": 1.48} warm_start_l = [ start1, start1, start1, start1, start1, start2, start2, start2, start3, start3, start3, start4, start4, start4, start5, start5, start5, ] n_iter = len(warm_start_l) hyper = Hyperactive() hyper.add_search( objective_function, search_space, n_iter=n_iter, initialize={"warm_start": warm_start_l}, early_stopping=early_stopping, ) hyper.run() search_data = hyper.results(objective_function) n_performed_iter = len(search_data) print("\n n_performed_iter \n", n_performed_iter) print("\n n_iter_no_change \n", n_iter_no_change) assert n_performed_iter == n_iter
def test_trafo_0(search_space): hyper = Hyperactive() hyper.add_search(objective_function_0, search_space, n_iter=25) hyper.run() for value in hyper.results(objective_function_0)["x1"].values: if value not in search_space["x1"]: assert False
def test_max_time_1(): c_time1 = time.time() hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=1000000) hyper.run(max_time=1) diff_time1 = time.time() - c_time1 assert 0.3 < diff_time1 < 2
def test_attributes_best_para_objective_function_0(): hyper = Hyperactive() hyper.add_search( objective_function, search_space, n_iter=15, ) hyper.run() assert isinstance(hyper.best_para(objective_function), dict)
def test_n_jobs_6(): hyper = Hyperactive() hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=2) hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=2) hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=2) hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=2) hyper.run() assert len(hyper.results_list) == 8