def test_shared_memory_1():
c_time = time.time()
hyper = Hyperactive(n_processes=1)
hyper.add_search(
model,
search_space,
n_iter=100,
n_jobs=1,
)
hyper.run()
d_time_1 = time.time() - c_time
c_time = time.time()
hyper = Hyperactive(n_processes=1)
hyper.add_search(
model,
search_space,
n_iter=100,
n_jobs=4,
)
hyper.run()
d_time_2 = time.time() - c_time
print("\n d_time_1 \n", d_time_1)
print("\n d_time_2 \n", d_time_2)
d_time_2 = d_time_2 / 2
assert d_time_1 / d_time_2 > 1.1
def test_memory_timeSave_1():
data = load_breast_cancer()
X, y = data.data, data.target
def objective_function(opt):
time.sleep(0.001)
return 1
search_space = {
"x1": list(np.arange(1, 30)),
}
results = pd.DataFrame(np.arange(1, 30), columns=["x1"])
results["score"] = 0
c_time1 = time.time()
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=1000,
memory_warm_start=results,
)
hyper.run()
diff_time1 = time.time() - c_time1
print("diff_time1", diff_time1)
assert diff_time1 < 1
def test_data_collector_2():
path = "./search_data.csv"
if os.path.isfile(path):
os.remove(path)
data_c = DataCollector(path, drop_duplicates=False)
def objective_function(opt):
score = -opt["x1"] * opt["x1"]
para_dict = {
"x1": opt["x1"],
"score": score,
}
data_c.append(para_dict)
return score
search_space = {
"x1": np.arange(-100, 101, 1),
}
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=50, memory=False)
hyper.run()
search_data = hyper.results(objective_function)[["x1", "score"]]
search_data_ = data_c.load()
assert df_equal(search_data, search_data_)
def test_shared_memory_warm_start_0():
c_time = time.time()
hyper = Hyperactive(n_processes=1)
hyper.add_search(
model,
search_space,
n_iter=100,
n_jobs=1,
)
hyper.run()
d_time_1 = time.time() - c_time
search_data0 = hyper.results(model)
c_time = time.time()
hyper = Hyperactive(n_processes=1)
hyper.add_search(
model,
search_space,
n_iter=100,
n_jobs=4,
memory_warm_start=search_data0,
)
hyper.run()
d_time_2 = time.time() - c_time
print("\n d_time_1 \n", d_time_1)
print("\n d_time_2 \n", d_time_2)
d_time_2 = d_time_2 / 2
assert d_time_2 * 1.4 - d_time_1 < 0
def test_data_collector_3():
if os.path.isfile(path):
os.remove(path)
search_space = {
"x1": np.arange(-100, 101, 0.001),
}
hyper = Hyperactive()
hyper.add_search(
objective_function_3,
search_space,
n_iter=5,
n_jobs=2,
memory=False,
initialize={"random": 1},
)
hyper.run()
search_data = hyper.results(objective_function_3)
search_data_ = data_c.load()
print("\n search_data \n", search_data)
print("\n search_data_ \n", search_data_)
assert df_equal(search_data, search_data_)
def test_search_space_3():
def func1():
pass
def func2():
pass
def func3():
pass
def objective_function(opt):
score = -opt["x1"] * opt["x1"]
return score
search_space = {
"x1": list(range(0, 100, 1)),
"func1": [func1, func2, func3],
}
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=15,
)
hyper.run()
assert isinstance(hyper.results(objective_function), pd.DataFrame)
assert (hyper.best_para(objective_function)["func1"]
in search_space["func1"])
def test_search_space_4():
class class1:
pass
class class2:
pass
class class3:
pass
def objective_function(opt):
score = -opt["x1"] * opt["x1"]
return score
search_space = {
"x1": list(range(0, 100, 1)),
"class1": [class1, class2, class3],
}
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=15,
)
hyper.run()
assert isinstance(hyper.results(objective_function), pd.DataFrame)
assert (hyper.best_para(objective_function)["class1"]
in search_space["class1"])
def test_attributes_results_6():
def objective_function(optimizer):
score = (-optimizer.suggested_params["x1"] *
optimizer.suggested_params["x1"])
return score
search_space = {
"x1": np.arange(0, 10, 1),
}
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=20,
initialize={"random": 1},
memory=False,
)
hyper.run()
x1_results = list(hyper.results(objective_function)["x1"].values)
print("\n x1_results \n", x1_results)
assert len(set(x1_results)) < len(x1_results)
def test_filter_data_0():
search_id1 = "test_model1"
search_id2 = "test_model2"
search_id3 = "test_model3"
search_ids = [search_id1, search_id2, search_id3]
def objective_function(opt):
score = -opt["x1"] * opt["x1"]
return score
search_space = {
"x1": np.arange(-100, 101, 1),
}
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=200)
hyper.run()
search_data = hyper.results(objective_function)
indices = list(search_space.keys()) + ["score"]
filter_dict = {
"parameter": indices,
"lower bound": "---",
"upper bound": "---",
}
filter_df = pd.DataFrame(filter_dict)
threshold = -1000
filter_df["lower bound"].iloc[1] = threshold
board = StreamlitBackend(search_ids)
progress_data = board.filter_data(search_data, filter_df)
assert not np.all(search_data["score"].values >= threshold)
assert np.all(progress_data["score"].values >= threshold)
def test_ltm_10():
path = None
model_name = "test_ltm_0"
def objective_function(opt):
score = -opt["x1"] * opt["x1"]
return score
search_space = {
"x1": list(range(0, 3, 1)),
}
memory = LongTermMemory(model_name, path=path)
hyper = Hyperactive()
hyper.add_search(
objective_function, search_space, n_iter=25, long_term_memory=memory
)
hyper.run()
hyper = Hyperactive()
hyper.add_search(
objective_function, search_space, n_iter=25, long_term_memory=memory
)
hyper.run()
memory.remove_model_data()
def test_issue_29():
data = load_boston()
X, y = data.data, data.target
def model(para):
gbr = GradientBoostingRegressor(
n_estimators=para["n_estimators"],
max_depth=para["max_depth"],
min_samples_split=para["min_samples_split"],
)
scores = cross_val_score(gbr, X, y, cv=3)
print(
"Iteration:",
para.optimizer.nth_iter,
" Best score",
para.optimizer.best_score,
)
return scores.mean()
search_space = {
"n_estimators": list(range(10, 150, 5)),
"max_depth": list(range(2, 12)),
"min_samples_split": list(range(2, 22)),
}
hyper = Hyperactive()
hyper.add_search(model, search_space, n_iter=20)
hyper.run()
def test_streamlit_backend_2():
search_id1 = "test_model1"
search_id2 = "test_model2"
search_id3 = "test_model3"
search_ids = [search_id1, search_id2, search_id3]
board = StreamlitBackend(search_ids)
def objective_function(opt):
score = -opt["x1"] * opt["x1"]
return score
search_space = {
"x1": np.arange(-100, 101, 1),
}
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=200)
hyper.run()
search_data = hyper.results(objective_function)
search_data["nth_iter"] = 0
search_data["score_best"] = 0
search_data["nth_process"] = 0
search_data["best"] = 0
plotly_fig = board.plotly(search_data, search_id1)
assert plotly_fig is not None
def test_ltm_0(objective_function, search_space, path):
(search_space, compare) = search_space
print("\n objective_function \n", objective_function)
print("\n search_space \n", search_space)
print("\n compare \n", compare)
print("\n path \n", path)
model_name = str(objective_function.__name__)
hyper = Hyperactive()
hyper.add_search(objective_function,
search_space,
n_iter=10,
initialize={"random": 1})
hyper.run()
results1 = hyper.results(objective_function)
memory = LongTermMemory(model_name, path=path)
memory.save(results1, objective_function)
results2 = memory.load()
print("\n results1 \n", results1)
print("\n results2 \n", results2)
memory.remove_model_data()
compare(results1, results2)
def test_early_stop_3():
def objective_function(para):
score = -para["x1"] * para["x1"]
return score
search_space = {
"x1": np.arange(0, 100, 0.1),
}
n_iter_no_change = 5
early_stopping = {
"n_iter_no_change": n_iter_no_change,
}
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=100000,
initialize={"warm_start": [{
"x1": 0
}]},
early_stopping=early_stopping,
)
hyper.run()
search_data = hyper.results(objective_function)
n_performed_iter = len(search_data)
print("\n n_performed_iter \n", n_performed_iter)
print("\n n_iter_no_change \n", n_iter_no_change)
assert n_performed_iter == (n_iter_no_change + 1)
def test_trafo_1(random_state, search_space, memory_warm_start):
search_space = search_space
memory_warm_start = memory_warm_start
c_time_0 = time.time()
hyper = Hyperactive()
hyper.add_search(
objective_function_1,
search_space,
n_iter=10,
random_state=random_state,
initialize={"random": 1},
)
hyper.run()
d_time_0 = time.time() - c_time_0
c_time_1 = time.time()
hyper = Hyperactive()
hyper.add_search(
objective_function_1,
search_space,
n_iter=10,
random_state=random_state,
initialize={"random": 1},
memory_warm_start=memory_warm_start,
)
hyper.run()
d_time_1 = time.time() - c_time_1
assert d_time_1 < d_time_0 * 0.5
def test_attributes_results_1():
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=100)
hyper.run()
assert set(search_space.keys()) < set(
hyper.results(objective_function).columns)
def test_max_score_1():
def objective_function(para):
score = -para["x1"] * para["x1"]
time.sleep(0.01)
return score
search_space = {
"x1": np.arange(0, 100, 0.1),
}
max_score = -9999
c_time = time.time()
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=100000,
initialize={"warm_start": [{
"x1": 99
}]},
max_score=max_score,
)
hyper.run()
diff_time = time.time() - c_time
print("\n Results head \n", hyper.results(objective_function).head())
print("\n Results tail \n", hyper.results(objective_function).tail())
print("\nN iter:", len(hyper.results(objective_function)))
assert diff_time < 1
def test_memory_warm_start_1(search_space):
c_time = time.time()
hyper0 = Hyperactive()
hyper0.add_search(objective_function,
search_space,
n_iter=2000,
memory=False)
hyper0.run()
d_time_1 = time.time() - c_time
search_data0 = hyper0.results(objective_function)
c_time = time.time()
hyper1 = Hyperactive()
hyper1.add_search(objective_function,
search_space,
n_iter=2000,
memory_warm_start=search_data0)
hyper1.run()
d_time_2 = time.time() - c_time
print("\n d_time_1 ", d_time_1)
print("\n d_time_2 ", d_time_2)
assert d_time_1 * 0.5 > d_time_2
def test_random_state_2():
hyper0 = Hyperactive()
hyper0.add_search(
objective_function,
search_space,
n_iter=10,
initialize={"random": 1},
random_state=1,
)
hyper0.run()
hyper1 = Hyperactive()
hyper1.add_search(
objective_function,
search_space,
n_iter=10,
initialize={"random": 1},
random_state=10,
)
hyper1.run()
best_score0 = hyper0.best_score(objective_function)
best_score1 = hyper1.best_score(objective_function)
assert abs(best_score0 - best_score1) > err
def test_n_processes_5():
hyper = Hyperactive(n_processes=1)
hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=4)
hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=4)
hyper.run()
assert len(hyper.results_list) == 8
def test_memory_timeSave_0():
data = load_breast_cancer()
X, y = data.data, data.target
def objective_function(opt):
dtc = DecisionTreeClassifier(
min_samples_split=opt["min_samples_split"])
scores = cross_val_score(dtc, X, y, cv=5)
return scores.mean()
search_space = {
"min_samples_split": np.arange(2, 20),
}
c_time1 = time.time()
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=100)
hyper.run()
diff_time1 = time.time() - c_time1
c_time2 = time.time()
hyper = Hyperactive()
hyper.add_search(objective_function,
search_space,
n_iter=100,
memory=False)
hyper.run()
diff_time2 = time.time() - c_time2
assert diff_time1 < diff_time2 * 0.8
def test_memory_timeSave_1():
data = load_breast_cancer()
X, y = data.data, data.target
def objective_function(opt):
dtc = DecisionTreeClassifier(max_depth=opt["max_depth"])
scores = cross_val_score(dtc, X, y, cv=5)
return scores.mean()
search_space = {
"max_depth": list(np.arange(1, 101)),
}
results = pd.DataFrame(np.arange(1, 101), columns=["max_depth"])
results["score"] = 0
c_time1 = time.time()
hyper = Hyperactive()
hyper.add_search(objective_function,
search_space,
n_iter=300,
memory_warm_start=results)
hyper.run()
diff_time1 = time.time() - c_time1
assert diff_time1 < 1
def test_memory_warm_start_manual():
data = load_breast_cancer()
X, y = data.data, data.target
def objective_function(opt):
dtc = GradientBoostingClassifier(n_estimators=opt["n_estimators"], )
scores = cross_val_score(dtc, X, y, cv=5)
return scores.mean()
search_space = {
"n_estimators": list(np.arange(500, 502)),
}
c_time_1 = time.time()
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=1)
hyper.run()
diff_time_1 = time.time() - c_time_1
memory_warm_start = pd.DataFrame([[500, 0.9], [501, 0.91]],
columns=["n_estimators", "score"])
c_time = time.time()
hyper0 = Hyperactive()
hyper0.add_search(
objective_function,
search_space,
n_iter=10,
memory_warm_start=memory_warm_start,
)
hyper0.run()
diff_time = time.time() - c_time
assert diff_time_1 > diff_time * 0.3
def test_attributes_result_errors_2():
with pytest.raises(ValueError):
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=15)
hyper.run()
hyper.results(objective_function1)
def test_max_score_0():
def objective_function(para):
score = -para["x1"] * para["x1"]
return score
search_space = {
"x1": np.arange(0, 100, 0.1),
}
max_score = -9999
opt = HillClimbingOptimizer(
epsilon=0.01,
rand_rest_p=0,
)
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
optimizer=opt,
n_iter=100000,
initialize={"warm_start": [{
"x1": 99
}]},
max_score=max_score,
)
hyper.run()
print("\n Results head \n", hyper.results(objective_function).head())
print("\n Results tail \n", hyper.results(objective_function).tail())
print("\nN iter:", len(hyper.results(objective_function)))
assert -100 > hyper.best_score(objective_function) > max_score
def test_early_stop_6():
def objective_function(para):
return para["x1"]
search_space = {
"x1": np.arange(0, 100, 0.01),
}
n_iter_no_change = 5
early_stopping = {
"n_iter_no_change": 5,
"tol_abs": None,
"tol_rel": 10,
}
start1 = {"x1": 1}
start2 = {"x1": 1.1}
start3 = {"x1": 1.22}
start4 = {"x1": 1.35}
start5 = {"x1": 1.48}
warm_start_l = [
start1,
start1,
start1,
start1,
start1,
start2,
start2,
start2,
start3,
start3,
start3,
start4,
start4,
start4,
start5,
start5,
start5,
]
n_iter = len(warm_start_l)
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=n_iter,
initialize={"warm_start": warm_start_l},
early_stopping=early_stopping,
)
hyper.run()
search_data = hyper.results(objective_function)
n_performed_iter = len(search_data)
print("\n n_performed_iter \n", n_performed_iter)
print("\n n_iter_no_change \n", n_iter_no_change)
assert n_performed_iter == n_iter
def test_trafo_0(search_space):
hyper = Hyperactive()
hyper.add_search(objective_function_0, search_space, n_iter=25)
hyper.run()
for value in hyper.results(objective_function_0)["x1"].values:
if value not in search_space["x1"]:
assert False
def test_max_time_1():
c_time1 = time.time()
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=1000000)
hyper.run(max_time=1)
diff_time1 = time.time() - c_time1
assert 0.3 < diff_time1 < 2
def test_attributes_best_para_objective_function_0():
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=15,
)
hyper.run()
assert isinstance(hyper.best_para(objective_function), dict)
def test_n_jobs_6():
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=2)
hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=2)
hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=2)
hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=2)
hyper.run()
assert len(hyper.results_list) == 8
