def evaluate(self, fnc, params=None, mode="train"):
partition = self.partitions[mode]
if params is not None:
wrapper = AlgoWrapper(fnc, params)
else:
wrapper = fnc
r = parallel_evaluate(
solvers=[wrapper],
task_subset=partition, # set to None to evaluate on all tasks
n_reps=128, # number of repetitions
eval_kwargs={'n_calls': 64},
joblib_kwargs={
'n_jobs': -1,
'verbose': 10
})
p = calculate_metrics(r) # returns pandas dataframe
# load the dataframe with existing results
df = pd.read_csv(csv_path)
# get the names of the functions that were actually used.
# it is assumed that these functions are present in the
# loaded csv as well.
names = [f.__name__ for f in partition]
df = df.set_index('Unnamed: 0')
# select only the names of tasks in partition
df = df.loc[names]
# insert found results
df[fnc.__name__] = p[fnc.__name__]
# drop index for proper compatibility with get_average_ranking function
df = df.reset_index()
rankings = get_average_ranking(df)
thisranking = rankings[fnc.__name__]
# want to minimize ranking. Less ranking means more performant algorithm
obj = thisranking / len(rankings)
print('rankings:', rankings)
print('objective:', thisranking)
return obj
"""
Example of running the benchmarks locally
on skopt and other software.
"""
from bbob.evaluation import parallel_evaluate, plot_results, calculate_metrics
from skopt import forest_minimize
from bbob.wrappers.gpyopt_minimize import gpyopt_minimize
from bbob.wrappers.hyperopt_minimize import hyperopt_minimize
from bbob.tracks.ampgo import Hartmann3_3_ri, Ackley_3_1_r
r = parallel_evaluate(
solvers=[forest_minimize, gpyopt_minimize, hyperopt_minimize],
task_subset=[Hartmann3_3_ri,
Ackley_3_1_r], # set to None to evaluate on all tasks
n_reps=2, # number of repetitions
eval_kwargs={'n_calls': 10},
joblib_kwargs={
'n_jobs': -1,
'verbose': 10
})
p = calculate_metrics(r) # returns pandas dataframe
p.to_csv('data.csv')
plot_results(r)
import sys
from skopt import gp_minimize
from bbob.wrappers.gpyopt_minimize import gpyopt_minimize
from bbob.wrappers.hyperopt_minimize import hyperopt_minimize
methods = [gp_minimize, gpyopt_minimize, hyperopt_minimize]
if sys.version[0] == '3':
from bbob.wrappers.smac_minimize import smac_minimize
methods += [smac_minimize]
if sys.version[0] == '2':
from bbob.wrappers.spearmint_minimize import spearmint_minimize
methods += [spearmint_minimize]
from bbob.evaluation import parallel_evaluate, calculate_metrics
from bbob.tracks import ampgo
r = parallel_evaluate(solvers=methods,
task_subset=[ampgo.Ackley_3_1_r],
n_reps=2,
joblib_kwargs={
'verbose': 10,
'n_jobs': 1
},
eval_kwargs={'n_calls': 10})
#plot_results(r)
m = calculate_metrics(r)
m.to_csv('data.csv')