def test_random_state():
opt0 = Hyperactive(X, y, random_state=False, memory=memory)
opt0.search(search_config)
opt1 = Hyperactive(X, y, random_state=0, memory=memory)
opt1.search(search_config)
opt2 = Hyperactive(X, y, random_state=1, memory=memory)
opt2.search(search_config)
def test_shared_memory_warm_start_2():
c_time = time.time()
hyper = Hyperactive(n_processes=1)
hyper.add_search(
model,
search_space,
n_iter=100,
n_jobs=1,
)
hyper.run()
d_time_1 = time.time() - c_time
search_data0 = hyper.results(model)
c_time = time.time()
hyper = Hyperactive(n_processes=1)
hyper.add_search(
model,
search_space,
n_iter=100,
n_jobs=1,
memory_warm_start=search_data0,
)
hyper.add_search(
model,
search_space,
n_iter=100,
n_jobs=1,
memory_warm_start=search_data0,
)
hyper.add_search(
model,
search_space,
n_iter=100,
n_jobs=1,
memory_warm_start=search_data0,
)
hyper.add_search(
model,
search_space,
n_iter=100,
n_jobs=1,
memory_warm_start=search_data0,
)
hyper.run()
d_time_2 = time.time() - c_time
print("\n d_time_1 \n", d_time_1)
print("\n d_time_2 \n", d_time_2)
d_time_2 = d_time_2 / 2
assert d_time_1 / d_time_2 > 1.1
def test_HillClimbingOptimizer():
opt = Hyperactive(X, y, memory=memory)
opt.search(search_config, n_iter=n_iter, optimizer="HillClimbing")
for epsilon in [0.01, 0.1, 1]:
opt = Hyperactive(X, y, memory=memory)
opt.search(
search_config,
n_iter=n_iter,
optimizer={"HillClimbing": {
"epsilon": epsilon
}},
)
def test_ParticleSwarmOptimizer():
opt = Hyperactive(X, y, memory=memory)
opt.search(search_config, n_iter=n_iter, optimizer="ParticleSwarm")
for n_particles in [2, 10, 30]:
opt = Hyperactive(X, y, memory=memory)
opt.search(
search_config,
n_iter=n_iter,
optimizer={"ParticleSwarm": {
"n_particles": n_particles
}},
)
def test_StochasticTunnelingOptimizer():
opt = Hyperactive(X, y, memory=memory)
opt.search(search_config, n_iter=n_iter, optimizer="StochasticTunneling")
for start_temp in [0.1, 1, 10]:
opt = Hyperactive(X, y, memory=memory)
opt.search(
search_config,
n_iter=n_iter,
optimizer={"StochasticTunneling": {
"start_temp": start_temp
}},
)
def test_ParallelTemperingOptimizer():
opt = Hyperactive(X, y, memory=memory)
opt.search(search_config, n_iter=n_iter, optimizer="ParallelTempering")
for n_swaps in [1, 10, 30]:
opt = Hyperactive(X, y, memory=memory)
opt.search(
search_config,
n_iter=n_iter,
optimizer={"ParallelTempering": {
"n_swaps": n_swaps
}},
)
def test_memory_warm_start_0(search_space):
hyper0 = Hyperactive()
hyper0.add_search(objective_function, search_space, n_iter=20)
hyper0.run()
search_data0 = hyper0.results(objective_function)
hyper1 = Hyperactive()
hyper1.add_search(objective_function,
search_space,
n_iter=20,
memory_warm_start=search_data0)
hyper1.run()
def test_RandomAnnealingOptimizer():
opt = Hyperactive(X, y, memory=memory)
opt.search(search_config, n_iter=n_iter, optimizer="RandomAnnealing")
for start_temp in [0.1, 1, 10]:
opt = Hyperactive(X, y, memory=memory)
opt.search(
search_config,
n_iter=n_iter,
optimizer={"RandomAnnealing": {
"start_temp": start_temp
}},
)
def test_TabuOptimizer():
opt = Hyperactive(X, y, memory=memory)
opt.search(search_config, n_iter=n_iter, optimizer="TabuSearch")
for tabu_memory in [1, 3, 5]:
opt = Hyperactive(X, y, memory=memory)
opt.search(
search_config,
n_iter=n_iter,
optimizer={"TabuSearch": {
"tabu_memory": tabu_memory
}},
)
def test_EvolutionStrategyOptimizer():
opt = Hyperactive(X, y, memory=memory)
opt.search(search_config, n_iter=n_iter, optimizer="EvolutionStrategy")
for individuals in [2, 10, 30]:
opt = Hyperactive(X, y, memory=memory)
opt.search(
search_config,
n_iter=n_iter,
optimizer={"EvolutionStrategy": {
"individuals": individuals
}},
)
def test_BayesianOptimizer():
opt = Hyperactive(X, y, memory=memory)
opt.search(search_config, n_iter=int(n_iter / 33), optimizer="Bayesian")
for warm_start_smbo in [True]:
opt = Hyperactive(X, y, memory="long")
opt.search(
search_config,
n_iter=int(n_iter / 33),
optimizer={"Bayesian": {
"warm_start_smbo": warm_start_smbo
}},
)
def test_Bayesian():
opt0 = Hyperactive(X, y, random_state=random_state)
opt0.search(search_config,
optimizer="Bayesian",
n_iter=n_iter_min,
init_config=init_config)
opt1 = Hyperactive(X, y, random_state=random_state)
opt1.search(search_config,
optimizer="Bayesian",
n_iter=n_iter_max,
init_config=init_config)
assert opt0.best_scores[model] < opt1.best_scores[model]
def test_issue_25():
# set a path to save the dataframe
path = "./search_data.csv"
search_space = {
"n_neighbors": list(range(1, 50)),
}
# get para names from search space + the score
para_names = list(search_space.keys()) + ["score"]
# init empty pandas dataframe
search_data = pd.DataFrame(columns=para_names)
search_data.to_csv(path, index=False)
def objective_function(para):
# score = random.choice([1.2, 2.3, np.nan])
score = np.nan
# you can access the entire dictionary from "para"
parameter_dict = para.para_dict
# save the score in the copy of the dictionary
parameter_dict["score"] = score
# append parameter dictionary to pandas dataframe
search_data = pd.read_csv(path, na_values="nan")
search_data_new = pd.DataFrame(parameter_dict,
columns=para_names,
index=[0])
search_data = search_data.append(search_data_new)
search_data.to_csv(path, index=False, na_rep="nan")
return score
hyper0 = Hyperactive()
hyper0.add_search(objective_function, search_space, n_iter=50)
hyper0.run()
search_data_0 = pd.read_csv(path, na_values="nan")
"""
the second run should be much faster than before,
because Hyperactive already knows most parameters/scores
"""
hyper1 = Hyperactive()
hyper1.add_search(objective_function,
search_space,
n_iter=50,
memory_warm_start=search_data_0)
hyper1.run()
def test_StochasticHillClimbingOptimizer():
opt = Hyperactive(X, y, memory=memory)
opt.search(search_config,
n_iter=n_iter,
optimizer="StochasticHillClimbing")
for p_down in [0.01, 0.1, 1]:
opt = Hyperactive(X, y, memory=memory)
opt.search(
search_config,
n_iter=n_iter,
optimizer={"StochasticHillClimbing": {
"p_down": p_down
}},
)
def test_memory_timeSave_1():
data = load_breast_cancer()
X, y = data.data, data.target
def objective_function(opt):
dtc = DecisionTreeClassifier(max_depth=opt["max_depth"])
scores = cross_val_score(dtc, X, y, cv=5)
return scores.mean()
search_space = {
"max_depth": list(np.arange(1, 101)),
}
results = pd.DataFrame(np.arange(1, 101), columns=["max_depth"])
results["score"] = 0
c_time1 = time.time()
hyper = Hyperactive()
hyper.add_search(objective_function,
search_space,
n_iter=300,
memory_warm_start=results)
hyper.run()
diff_time1 = time.time() - c_time1
assert diff_time1 < 1
def meta_opt(para, X_list, y_list):
scores = []
for X, y in zip(X_list, y_list):
X_list, y_list = data_aug(X, y, sample_multi=3, feature_multi=3)
for X, y in zip(X_list, y_list):
for n_iter in [10, 25, 50, 100]:
opt = Hyperactive(
search_config_model,
optimizer={
"ParticleSwarm": {
"inertia": para["inertia"],
"cognitive_weight": para["cognitive_weight"],
"social_weight": para["social_weight"],
}
},
n_iter=n_iter,
verbosity=None,
)
opt.search(X, y)
score = opt.score_best
scores.append(score)
return np.array(scores).mean()
def meta_opt(para, X, y):
def model(para, X, y):
model = DecisionTreeClassifier(
max_depth=para["max_depth"],
min_samples_split=para["min_samples_split"],
min_samples_leaf=para["min_samples_leaf"],
)
scores = cross_val_score(model, X, y, cv=3)
return scores.mean()
search_config = {
model: {
"max_depth": range(2, 50),
"min_samples_split": range(2, 50),
"min_samples_leaf": range(1, 50),
}
}
opt = Hyperactive(
search_config,
optimizer={
"ParticleSwarm": {
"inertia": para["inertia"],
"cognitive_weight": para["cognitive_weight"],
"social_weight": para["social_weight"],
}
},
verbosity=None,
)
opt.search(X, y)
return opt.score_best
def test_gpr():
opt = Hyperactive(X, y, memory=memory)
opt.search(search_config,
n_iter=n_iter,
optimizer={"Bayesian": {
"gpr": "gp_linear"
}})
def test_attributes_result_errors_2():
with pytest.raises(ValueError):
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=15)
hyper.run()
hyper.results(objective_function1)
def test_ltm_0(objective_function, search_space, path):
(search_space, compare) = search_space
print("\n objective_function \n", objective_function)
print("\n search_space \n", search_space)
print("\n compare \n", compare)
print("\n path \n", path)
model_name = str(objective_function.__name__)
hyper = Hyperactive()
hyper.add_search(objective_function,
search_space,
n_iter=10,
initialize={"random": 1})
hyper.run()
results1 = hyper.results(objective_function)
memory = LongTermMemory(model_name, path=path)
memory.save(results1, objective_function)
results2 = memory.load()
print("\n results1 \n", results1)
print("\n results2 \n", results2)
memory.remove_model_data()
compare(results1, results2)
def test_attributes_results_1():
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=100)
hyper.run()
assert set(search_space.keys()) < set(
hyper.results(objective_function).columns)
def test_sklearn():
from sklearn.tree import DecisionTreeClassifier
def model(para, X_train, y_train):
model = DecisionTreeClassifier(
criterion=para["criterion"],
max_depth=para["max_depth"],
min_samples_split=para["min_samples_split"],
min_samples_leaf=para["min_samples_leaf"],
)
scores = cross_val_score(model, X_train, y_train, cv=3)
return scores.mean()
search_config = {
model: {
"criterion": ["gini", "entropy"],
"max_depth": range(1, 21),
"min_samples_split": range(2, 21),
"min_samples_leaf": range(1, 21),
}
}
opt = Hyperactive(X, y)
opt.search(search_config)
def test_streamlit_backend_2():
search_id1 = "test_model1"
search_id2 = "test_model2"
search_id3 = "test_model3"
search_ids = [search_id1, search_id2, search_id3]
board = StreamlitBackend(search_ids)
def objective_function(opt):
score = -opt["x1"] * opt["x1"]
return score
search_space = {
"x1": np.arange(-100, 101, 1),
}
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=200)
hyper.run()
search_data = hyper.results(objective_function)
search_data["nth_iter"] = 0
search_data["score_best"] = 0
search_data["nth_process"] = 0
search_data["best"] = 0
plotly_fig = board.plotly(search_data, search_id1)
assert plotly_fig is not None
def test_early_stop_3():
def objective_function(para):
score = -para["x1"] * para["x1"]
return score
search_space = {
"x1": np.arange(0, 100, 0.1),
}
n_iter_no_change = 5
early_stopping = {
"n_iter_no_change": n_iter_no_change,
}
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=100000,
initialize={"warm_start": [{
"x1": 0
}]},
early_stopping=early_stopping,
)
hyper.run()
search_data = hyper.results(objective_function)
n_performed_iter = len(search_data)
print("\n n_performed_iter \n", n_performed_iter)
print("\n n_iter_no_change \n", n_iter_no_change)
assert n_performed_iter == (n_iter_no_change + 1)
def test_max_score_0():
def objective_function(para):
score = -para["x1"] * para["x1"]
return score
search_space = {
"x1": np.arange(0, 100, 0.1),
}
max_score = -9999
opt = HillClimbingOptimizer(
epsilon=0.01,
rand_rest_p=0,
)
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
optimizer=opt,
n_iter=100000,
initialize={"warm_start": [{
"x1": 99
}]},
max_score=max_score,
)
hyper.run()
print("\n Results head \n", hyper.results(objective_function).head())
print("\n Results tail \n", hyper.results(objective_function).tail())
print("\nN iter:", len(hyper.results(objective_function)))
assert -100 > hyper.best_score(objective_function) > max_score
def test_filter_data_0():
search_id1 = "test_model1"
search_id2 = "test_model2"
search_id3 = "test_model3"
search_ids = [search_id1, search_id2, search_id3]
def objective_function(opt):
score = -opt["x1"] * opt["x1"]
return score
search_space = {
"x1": np.arange(-100, 101, 1),
}
hyper = Hyperactive()
hyper.add_search(objective_function, search_space, n_iter=200)
hyper.run()
search_data = hyper.results(objective_function)
indices = list(search_space.keys()) + ["score"]
filter_dict = {
"parameter": indices,
"lower bound": "---",
"upper bound": "---",
}
filter_df = pd.DataFrame(filter_dict)
threshold = -1000
filter_df["lower bound"].iloc[1] = threshold
board = StreamlitBackend(search_ids)
progress_data = board.filter_data(search_data, filter_df)
assert not np.all(search_data["score"].values >= threshold)
assert np.all(progress_data["score"].values >= threshold)
def test_n_processes_5():
hyper = Hyperactive(n_processes=1)
hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=4)
hyper.add_search(objective_function, search_space, n_iter=15, n_jobs=4)
hyper.run()
assert len(hyper.results_list) == 8
def test_warm_start_smbo():
opt = Hyperactive(X, y, memory="long")
opt.search(search_config,
n_iter=n_iter,
optimizer={"Bayesian": {
"warm_start_smbo": True
}})
def test_max_score_1():
def objective_function(para):
score = -para["x1"] * para["x1"]
time.sleep(0.01)
return score
search_space = {
"x1": np.arange(0, 100, 0.1),
}
max_score = -9999
c_time = time.time()
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=100000,
initialize={"warm_start": [{
"x1": 99
}]},
max_score=max_score,
)
hyper.run()
diff_time = time.time() - c_time
print("\n Results head \n", hyper.results(objective_function).head())
print("\n Results tail \n", hyper.results(objective_function).tail())
print("\nN iter:", len(hyper.results(objective_function)))
assert diff_time < 1
def test_attributes_results_6():
def objective_function(optimizer):
score = (-optimizer.suggested_params["x1"] *
optimizer.suggested_params["x1"])
return score
search_space = {
"x1": np.arange(0, 10, 1),
}
hyper = Hyperactive()
hyper.add_search(
objective_function,
search_space,
n_iter=20,
initialize={"random": 1},
memory=False,
)
hyper.run()
x1_results = list(hyper.results(objective_function)["x1"].values)
print("\n x1_results \n", x1_results)
assert len(set(x1_results)) < len(x1_results)
