def run(solver_object, budget=budget,
max_runs=max_runs,
current_batch=current_batch,
number_of_batches=number_of_batches):
"""Initialize suite and observer, then benchmark solver by calling
`batch_loop(SOLVER, suite, observer, budget,...`.
"""
##############################################################################
solver = solver_object.ibea
suite_name = "bbob-biobj"
suite_instance = "year:2016"
suite_options = "dimensions: 2,3,5" # "dimensions: 2,3,5,10,20 " # if 40 is not desired
observer_name = suite_name
observer_options = (
' result_folder: %s_on_%s_budget%04dxD '
% (str(solver_object), suite_name, budget) +
' algorithm_name: %s ' % solver.__name__ +
' algorithm_info: "Indicator-based Evolutionary Algorithm (epsilon)" ')
observer = Observer(observer_name, observer_options)
suite = Suite(suite_name, suite_instance, suite_options)
print("Benchmarking solver '%s' with budget=%d*dimension on %s suite, %s"
% (' '.join(str(solver).split()[:2]), budget,
suite.name, time.asctime()))
if number_of_batches > 1:
print('Batch usecase, make sure you run *all* %d batches.\n' %
number_of_batches)
t0 = time.clock()
batch_loop(solver, suite, observer, budget, max_runs,
current_batch, number_of_batches)
print(", %s (%s total elapsed time)." % (time.asctime(), ascetime(time.clock() - t0)))
def main(budget=budget,
max_runs=max_runs,
current_batch=current_batch,
number_of_batches=number_of_batches):
"""Initialize suite and observer, then benchmark solver by calling
``batch_loop(SOLVER, suite, observer, budget,...``
"""
suite = Suite(suite_name, suite_instance, suite_options)
observer_name = default_observers()[suite_name]
# observer_name = another observer if so desired
observer_options.update_gracefully(default_observer_options())
observer = Observer(observer_name, observer_options.as_string)
print("Benchmarking solver '%s' with budget=%d*dimension on %s suite, %s" %
(' '.join(
str(SOLVER).split()[:2]), budget, suite.name, time.asctime()))
if number_of_batches > 1:
print('Batch usecase, make sure you run *all* %d batches.\n' %
number_of_batches)
t0 = time.clock()
batch_loop(SOLVER, suite, observer, budget, max_runs, current_batch,
number_of_batches)
print(", %s (%s total elapsed time)." %
(time.asctime(), ascetime(time.clock() - t0)))
print('Data written to folder', observer.result_folder)
print('To post-process the data call \n'
' python -m cocopp %s \n'
'from a system shell or \n'
' cocopp.main("%s") \n'
'from a python shell' % (2 * (observer.result_folder, )))
def main(budget=budget,
max_runs=max_runs,
current_batch=current_batch,
number_of_batches=number_of_batches):
"""Initialize suite and observer, then benchmark solver by calling
``batch_loop(SOLVER, suite, observer, budget,...``
"""
suite = Suite(suite_name, suite_instance, suite_options)
observer_name = default_observers()[suite_name]
# observer_name = another observer if so desired
# observer_options.update_gracefully(default_observer_options())
observer = Observer(observer_name, observer_options.as_string)
"""
print("Benchmarking solver '%s' with budget=%d*dimension on %s suite, %s"
% (' '.join(str(SOLVER).split()[:2]), budget,
suite.name, time.asctime()))
if number_of_batches > 1:
print('Batch usecase, make sure you run *all* %d batches.\n' %
number_of_batches)
t0 = time.clock()
"""
batch_loop(SOLVER, suite, observer, budget, max_runs, current_batch,
number_of_batches)
"""
def main(problem_subset,
budget,
max_runs=max_runs,
current_batch=current_batch,
number_of_batches=number_of_batches):
"""Initialize suite and observer, then benchmark solver by calling
``batch_loop(SOLVER, suite, observer, budget,...``
"""
observer_name = default_observers()[suite_name]
observer_options.update_gracefully(default_observer_options())
observer = Observer(observer_name, observer_options.as_string)
suite = Suite(suite_name, suite_instance, suite_options)
print("Benchmarking solver '%s' with budget=%d*dimension on %s suite, %s" %
(' '.join(
str(SOLVER).split()[:2]), budget, suite.name, time.asctime()))
if number_of_batches > 1:
print('Batch usecase, make sure you run *all* %d batches.\n' %
number_of_batches)
t0 = time.clock()
batch_loop(SOLVER,
suite,
observer,
budget,
max_runs,
current_batch,
number_of_batches,
problem_subset=problem_subset)
print(", %s (%s total elapsed time)." %
(time.asctime(), ascetime(time.clock() - t0)))
def main(nb_trials=15, budget_per_dim=100, output="benchmark.csv"):
suite_instance = "year:2016"
suite_name = "bbob"
suite_options = ""
suite = Suite(suite_name, suite_instance, suite_options)
algos = [random_search, cma, ucb]
stats = []
for i, fun in enumerate(suite):
print("Function {}".format(fun.name))
for algo in algos:
algo_name = algo.__name__
print('Algo : "{}"'.format(algo_name))
for trial in range(nb_trials):
print("Running trial {}...".format(trial + 1))
t0 = time.time()
xbest, ybest, nbeval = algo(fun,
budget_per_dim * fun.dimension)
delta_t = time.time() - t0
stats.append({
"func": fun.id,
"algo": algo_name,
"nbeval": nbeval,
"ybest": ybest,
"duration": delta_t,
})
stats = pd.DataFrame(stats)
stats.to_csv(output, index=False)
def process_test_cases(fd, suite_name, test_vectors):
"""
Read test cases for benchmark suite ${suite_name} from ${fd} and evaluate them.
"""
number_of_testcases = 0
number_of_failures = 0
previous_problem_index = None
suite = Suite(suite_name, "instances:1-15", "")
print("Testing suite", suite_name)
for test_case in fd:
number_of_testcases += 1
## A test case is a 4-tuple (deprecated_problem_index, problem_index, test_vector_id,
## expected_y) separated by a tab.
deprecated_problem_index, problem_index, test_vector_id, expected_y = test_case.split("\t")
## Do type conversion. Python gurus probably know an elegant
## one line solution...
deprecated_problem_index = int(deprecated_problem_index)
test_vector_id = int(test_vector_id)
expected_y = float(expected_y)
## We cache the problem instances because creating an instance
## can be expensive depending on the transformation.
if deprecated_problem_index != previous_problem_index:
problem = suite.get_problem(int(problem_index))
previous_problem_index = deprecated_problem_index
test_vector = test_vectors[test_vector_id]
y = problem(test_vector[: problem.number_of_variables])
if not about_equal(y, expected_y):
number_of_failures += 1
if number_of_failures < 100:
print(
"%8i %8i FAILED expected=%.8e observed=%.8e"
% (deprecated_problem_index, test_vector_id, expected_y, y)
)
elif number_of_failures == 100:
print("... further failed tests suppressed ...")
print(
"%i of %i tests passed (failure rate %.2f%%)"
% (
number_of_testcases - number_of_failures,
number_of_testcases,
(100.0 * number_of_failures) / number_of_testcases,
)
)
if number_of_failures > 0:
sys.exit(-1)
def process_test_cases(fd, suite_name, test_vectors):
"""
Read test cases for benchmark suite ${suite_name} from ${fd} and evaluate them.
"""
number_of_testcases = 0
number_of_failures = 0
previous_problem_index = None
suite = Suite(suite_name, "instances:1-15", "")
print("Testing suite", suite_name)
for test_case in fd:
number_of_testcases += 1
## A test case is a 4-tuple (deprecated_problem_index, problem_index, test_vector_id,
## expected_y) separated by a tab.
deprecated_problem_index, problem_index, test_vector_id, expected_y = test_case.split(
)
## Do type conversion. Python gurus probably know an elegant
## one line solution...
problem_index = int(problem_index)
test_vector_id = int(test_vector_id)
expected_y = float(expected_y)
## We cache the problem instances because creating an instance
## can be expensive depending on the transformation.
if problem_index != previous_problem_index:
problem = suite.get_problem(int(problem_index))
previous_problem_index = problem_index
test_vector = test_vectors[test_vector_id]
y = problem(test_vector[:problem.number_of_variables])
if not about_equal(y, expected_y, 4e-6):
number_of_failures += 1
if number_of_failures < 100:
print("%8i %8i FAILED expected=%.8e observed=%.8e" %
(problem_index, test_vector_id, expected_y, y))
elif number_of_failures == 100:
print("... further failed tests suppressed ...")
print("%i of %i tests passed (failure rate %.2f%%)" %
(number_of_testcases - number_of_failures, number_of_testcases,
(100.0 * number_of_failures) / number_of_testcases))
if number_of_failures > 0:
sys.exit(-1)
def run_constrained_suite_test():
from collections import defaultdict
try:
suite = Suite('bbob-constrained', '', '')
except NameError:
return
counts = defaultdict(int)
for f in suite:
counts[-5] += np.any(f.initial_solution < -5)
counts[5] += np.any(f.initial_solution > 5)
counts['c'] += np.any(f.constraint(f.initial_solution) > 0)
counts['b'] += np.any(
f.constraint(best_parameter(f)) >
1e-11) # mac: 6.8361219664552603e-12 is the largest value
assert sum(counts.values()) == 0
def run(self, budget=100, current_batch=1, number_of_batches=15):
# type: (int, int, int) -> None
suite = Suite(self._suite_name, self._suite_instance,
self._suite_options)
observer_name = default_observers()[self._suite_name]
observer_options = self._build_observer_options(budget)
observer = Observer(observer_name, observer_options.as_string)
for problem_index, problem in enumerate(tqdm(suite)):
if (problem_index % number_of_batches) != current_batch - 1:
continue
observer.observe(problem)
max_evals = budget * problem.dimension
self._solver(problem,
problem.lower_bounds,
problem.upper_bounds,
max_evals - problem.evaluations_constraints,
verbose=False)
try:
range = xrange # let range always be an iterator
except NameError:
pass
import numpy as np # "pip install numpy" installs numpy
import cocoex
from cocoex import Suite, Observer, log_level
verbose = 1
import bbobbenchmarks as bm
#functions = np.arange(1,6)
#instances = np.arange(1,10)
#dim = [2,3,5,10,20,40,80]
suite = Suite('bbob', 'year:2010', '')
numberOfDifferences = 0
largestAbsDifference = 0
largestRelDifference = 0
numberOfProblems = 0
numberOfSearchPoints = 0
for problem_index, problem in enumerate(suite):
f = int(problem.id.lower().split('_f')[1].split('_')[0])
d = int(problem.id.lower().split('_d')[1].split('_')[0])
i = int(problem.id.lower().split('_i')[1].split('_')[0])
numberOfProblems = numberOfProblems + 1
xrand = -4 + 8 * np.random.rand(d)
def log_reconstruct(input_path, output_path, algorithm_name, algorithm_info,
functions, instances, dimensions):
"""Reconstructs the .info, .dat and .tdat files produced by the logger from the .adat files in the input_path.
Takes into account only the given functions, instances and dimensions. If any .info, .dat and .tdat files of
the same names already exist in the output_path, the new data is appended to them.
"""
suite_name = 'bbob-biobj'
print('Reading archive information...')
archive_info = ArchiveInfo(input_path, functions, instances, dimensions,
False)
function_string = archive_info.get_function_string()
instance_string = archive_info.get_instance_string()
dimension_string = archive_info.get_dimension_string()
file_name_set = archive_info.get_file_name_set()
print('Initializing the suite and observer...')
suite_instance = 'instances: {}'.format(instance_string)
suite_options = 'dimensions: {} function_indices: {}'.format(
dimension_string, function_string)
suite = Suite(suite_name, suite_instance, suite_options)
observer_options = 'result_folder: {} algorithm_name: {} algorithm_info: "{}" log_nondominated: read'. \
format(output_path, algorithm_name, algorithm_info)
observer = Observer(suite_name, observer_options)
print('Reconstructing...')
for input_file in file_name_set:
(_suite_name, function, _instance,
dimension) = parse_archive_file_name(input_file)
with open(input_file, 'r') as f_in:
print(input_file)
problem = None
objective_vector = None
evaluation_found = False
instance = None
count_not_updated = 0
evaluation = 0
for line in f_in:
if len(line.split()) < 3:
continue
elif line[0] == '%' and 'instance' in line:
instance = int(get_key_value(line[1:], 'instance'))
if instance in instances:
if problem is not None:
if not evaluation_found:
raise PreprocessingWarning(
'Missing the line `% evaluations = ` in the previous '
'problem. This problem is file = {}, instance = {}'
.format(input_file, instance))
if count_not_updated > 0:
print(
'{} solutions did not update the archive'.
format(count_not_updated))
problem.free()
problem = suite.get_problem_by_function_dimension_instance(
function, dimension, instance, observer)
evaluation_found = False
elif line[0] != '%' and instance in instances:
try:
split = line.split()
evaluation = int(split[0])
objective_vector = np.array(split[1:3])
updated = problem.logger_biobj_feed_solution(
evaluation, objective_vector)
if updated == 0:
count_not_updated += 1
except ValueError as error:
print('Problem in file {}, line {}, skipping line\n{}'.
format(input_file, line, error))
continue
elif line[0] == '%' and 'evaluations' in line:
old_evaluation = evaluation
evaluation = int(get_key_value(line[1:], 'evaluations'))
evaluation_found = True
if (
evaluation > old_evaluation
) and problem is not None and objective_vector is not None:
problem.logger_biobj_feed_solution(
evaluation, objective_vector)
if problem is not None:
if not evaluation_found:
print(
'Missing the line `% evaluations = ` in this or the previous problem. This is file = {}, '
'instance = {}'.format(input_file, instance))
if count_not_updated > 0:
print('{} solutions did not update the archive'.format(
count_not_updated))
problem.free()
f_in.close()
def log_reconstruct(input_path, output_path, algorithm_name, algorithm_info, functions, instances, dimensions):
"""Reconstructs the .info, .dat and .tdat files produced by the logger from the .adat files in the input_path.
Takes into account only the given functions, instances and dimensions. If any .info, .dat and .tdat files of
the same names already exist in the output_path, the new data is appended to them.
"""
ext_suite_name = 'bbob-biobj-ext'
suite_name = 'bbob-biobj'
print('Reading archive information...')
archive_info = ArchiveInfo(input_path, functions, instances, dimensions, False)
function_string = archive_info.get_function_string()
instance_string = archive_info.get_instance_string()
dimension_string = archive_info.get_dimension_string()
file_name_set = archive_info.get_file_name_set()
print('Initializing the suite and observer...')
suite_instance = 'instances: {}'.format(instance_string)
suite_options = 'dimensions: {} function_indices: {}'.format(dimension_string, function_string)
suite = Suite(ext_suite_name, suite_instance, suite_options)
observer_options = 'result_folder: {} algorithm_name: {} algorithm_info: "{}" log_nondominated: read'. \
format(output_path, algorithm_name, algorithm_info)
observer = Observer(suite_name, observer_options)
print('Reconstructing...')
for input_file in file_name_set:
(_suite_name, function, _instance, dimension) = parse_archive_file_name(input_file)
with open(input_file, 'r') as f_in:
print(input_file)
problem = None
objective_vector = None
evaluation_found = False
instance = None
count_not_updated = 0
evaluation = 0
for line in f_in:
if len(line.split()) < 3:
continue
elif line[0] == '%' and 'instance' in line:
instance = int(get_key_value(line[1:], 'instance'))
if instance in instances:
if problem is not None:
if not evaluation_found:
raise PreprocessingWarning('Missing the line `% evaluations = ` in the previous '
'problem. This problem is file = {}, instance = {}'
.format(input_file, instance))
if count_not_updated > 0:
print('{} solutions did not update the archive'.format(count_not_updated))
problem.free()
problem = suite.get_problem_by_function_dimension_instance(function, dimension, instance,
observer)
evaluation_found = False
elif line[0] != '%' and instance in instances:
try:
split = line.split()
evaluation = int(split[0])
objective_vector = np.array(split[1:3])
updated = problem.logger_biobj_feed_solution(evaluation, objective_vector)
if updated == 0:
count_not_updated += 1
except ValueError as error:
print('Problem in file {}, line {}, skipping line\n{}'.format(input_file, line, error))
continue
elif line[0] == '%' and 'evaluations' in line:
old_evaluation = evaluation
evaluation = int(get_key_value(line[1:], 'evaluations'))
evaluation_found = True
if (evaluation > old_evaluation) and problem is not None and objective_vector is not None:
problem.logger_biobj_feed_solution(evaluation, objective_vector)
if problem is not None:
if not evaluation_found:
print('Missing the line `% evaluations = ` in this or the previous problem. This is file = {}, '
'instance = {}' .format(input_file, instance))
if count_not_updated > 0:
print('{} solutions did not update the archive'.format(count_not_updated))
problem.free()
f_in.close()
functions = range(1,56)
#functions = (1,2,3,10,20,30,54,55)
instances = (1,)
# Note: in single-objective bbobdocfunctions.pdf documentation, '0' seems to be the instance used
#inputfolderforParetoFronts = 'archives/before_workshop/archives/'
inputfolderforParetoFronts = 'F:/5D-archives-from-Tea/after_workshop/archives/'
#outputfolder = 'plots/before_workshop/'
outputfolder = 'plots/after_workshop/'
tofile = True # if True: files are written; if False: no files but screen output
###########################################
suite_name = "bbob-biobj"
suite_instance = "year:2016"
suite_options = "dimensions: 2,3,5,10,20,40"
suite = Suite(suite_name, suite_instance, suite_options)
for problem_index, problem in enumerate(suite):
f = int(problem.id.lower().split('_f')[1].split('_')[0])
d = int(problem.id.lower().split('_d')[1].split('_')[0])
i = int(problem.id.lower().split('_i')[1].split('_')[0])
f1_id = int(problem.name.lower().split('_f')[1].split('_')[0])
f2_id = int(problem.name.lower().split('_f')[2].split('_')[0])
i1 = int(problem.name.lower().split('_i')[1].split('_')[0])
i2 = int(problem.name.lower().split('_i')[2].split('_')[0])
if ((i not in instances) or (f not in functions)
or (d not in dims)):
"""
suite: dataset
observer: output
solver: optimisation of function
dimension, current_batch, number_of_batches: choose data
max_run: max times of run
"""
for fun_index, fun in enumerate(suite):
if (fun_index + current_batch - 1) % number_of_batches:
continue
if fun.dimension > dimension:
continue
fun.observe_with(observer)
solver(fun, fun.lower_bounds, fun.upper_bounds, budget)
# cocopp.main(observer.result_folder) # re-run folders look like "...-001" etc
# webbrowser.open("file://" + os.getcwd() + "/ppdata/index.html")
if __name__ == '__main__':
"""add parameters here """
suite = Suite('bbob-biobj', 'year:2018', '')
observer = Observer(
"bbob-biobj",
"result_folder: %s_on_%s" % (solver.__name__, "bbob2018"))
solver = random_search
dimension = 5
budget = 100
main(budget, suite, observer, solver, dimension, max_runs, current_batch,
number_of_batches)
