def test_update(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1)
}
experiment = Experiment(name="test_optimizer_experiment",
parameter_definitions=param_def)
self.optimizer.update(experiment)
def setup(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1)
}
experiment = Experiment(name="test_optimizer_experiment",
parameter_definitions=param_def)
self.optimizer = self.OptimizerStub(experiment, optimizer_params=None)
def test_check_optimizer(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1)
}
experiment = Experiment(name="test_optimizer_experiment",
parameter_definitions=param_def)
assert_is_instance(check_optimizer(RandomSearch, experiment, {"multiprocessing": "none"}), RandomSearch)
assert_is_instance(check_optimizer("RandomSearch", experiment, {"multiprocessing": "none"}), RandomSearch)
with assert_raises(ValueError):
check_optimizer("fails", experiment, {"multiprocessing": "none"})
with assert_raises(ValueError):
check_optimizer(MinMaxNumericParamDef, experiment)
queue_based = check_optimizer(RandomSearch, experiment,
{"multiprocessing": "queue"})
assert_is_instance(queue_based, QueueBasedOptimizer)
assert_equal(check_optimizer(queue_based, experiment,
{"multiprocessing": "queue"}), queue_based)
queue_based.exit()
assert_is_instance(check_optimizer(RandomSearch, experiment,
{"multiprocessing": "none"}),
RandomSearch)
with assert_raises(ValueError):
check_optimizer(RandomSearch, experiment,
{"multiprocessing": "fails"}),
time.sleep(0.1)
def test_init(self):
#test default parameters
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
opt = BayesianOptimizer(exp)
assert_equal(opt.initial_random_runs, 10)
assert_is_none(opt.acquisition_hyperparams)
assert_equal(opt.num_gp_restarts, 10)
assert_true(isinstance(opt.acquisition_function, ExpectedImprovement))
assert_dict_equal(opt.kernel_params, {})
assert_equal(opt.kernel, "matern52")
#test correct initialization
opt_arguments = {
"initial_random_runs": 5,
"acquisition_hyperparams": {},
"num_gp_restarts": 5,
"acquisition": ProbabilityOfImprovement,
"kernel_params": {},
"kernel": "matern52",
"mcmc": True,
}
opt = BayesianOptimizer(exp, opt_arguments)
assert_equal(opt.initial_random_runs, 5)
assert_dict_equal(opt.acquisition_hyperparams, {})
assert_equal(opt.num_gp_restarts, 5)
assert_true(
isinstance(opt.acquisition_function, ProbabilityOfImprovement))
assert_dict_equal(opt.kernel_params, {})
assert_equal(opt.kernel, "matern52")
def test_update_param_support(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"not_supported": OrdinalParamDef(["A", "B"])
}
experiment = Experiment(name="test_optimizer_experiment_crash",
parameter_definitions=param_def)
assert_raises(ValueError, self.optimizer.update, experiment)
def setup(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1)
}
experiment = Experiment(name="test_optimizer_experiment",
parameter_definitions=param_def)
self.optimizer = QueueBasedOptimizer(RandomSearch,
experiment)
def __init__(self, name, optimizer, param_defs, experiment=None, optimizer_arguments=None,
minimization=True, write_directory_base="/tmp/APSIS_WRITING",
experiment_directory_base=None, csv_write_frequency=1):
"""
Initializes the BasicExperimentAssistant.
Parameters
----------
name : string
The name of the experiment. This does not have to be unique, but is
for human orientation.
optimizer : Optimizer instance or string
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
Alternatively, it can be a string corresponding to the optimizer,
as defined by apsis.utilities.optimizer_utils.
param_defs : dict of ParamDef.
This is the parameter space defining the experiment.
experiment : Experiment
Preinitialize this assistant with an existing experiment.
optimizer_arguments=None : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
minimization=True : bool
Whether the problem is one of minimization or maximization.
write_directory_base : string, optional
The global base directory for all writing. Will only be used
for creation of experiment_directory_base if this is not given.
experiment_directory_base : string or None, optional
The directory to write all the results to. If not
given a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int, optional
States how often the csv file should be written to.
If set to 0 no results will be written.
"""
self.logger = get_logger(self)
self.logger.info("Initializing experiment assistant.")
self.optimizer = optimizer
self.optimizer_arguments = optimizer_arguments
if experiment is None:
self.experiment = Experiment(name, param_defs, minimization)
else:
self.experiment = experiment
self.csv_write_frequency = csv_write_frequency
if self.csv_write_frequency != 0:
self.write_directory_base = write_directory_base
if experiment_directory_base is not None:
self.experiment_directory_base = experiment_directory_base
ensure_directory_exists(self.experiment_directory_base)
else:
self._create_experiment_directory()
self.logger.info("Experiment assistant successfully initialized.")
def setup(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1)
}
self.experiment = Experiment(name="test_optimizer_experiment",
parameter_definitions=param_def)
out_queue = Queue()
in_queue = Queue()
self.backend = QueueBackend(RandomSearch, self.experiment, out_queue,
in_queue)
def test_get_next_candidate(self):
opt = RandomSearch({"initial_random_runs": 3})
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)}, NominalParamDef(["A", "B", "C"]))
for i in range(5):
cand = opt.get_next_candidates(exp)[0]
assert_true(isinstance(cand, Candidate))
cand.result = 2
exp.add_finished(cand)
cands = opt.get_next_candidates(exp, num_candidates=3)
assert_equal(len(cands), 3)
def test_EI(self):
opt = SimpleBayesianOptimizer({"initial_random_runs": 3})
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
for i in range(10):
cand = opt.get_next_candidates(exp)[0]
assert_true(isinstance(cand, Candidate))
cand.result = 2
exp.add_finished(cand)
cands = opt.get_next_candidates(exp, num_candidates=3)
assert_equal(len(cands), 3)
def test_PoI(self):
opt = SimpleBayesianOptimizer({"initial_random_runs": 3, "acquisition": ProbabilityOfImprovement})
assert_true(isinstance(opt.acquisition_function, ProbabilityOfImprovement))
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
for i in range(10):
cand = opt.get_next_candidates(exp)[0]
assert_true(isinstance(cand, Candidate))
cand.result = 2
exp.add_finished(cand)
cands = opt.get_next_candidates(exp, num_candidates=3)
assert_equal(len(cands), 3)
def test_warp(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1})
cand_out = exp.warp_pt_out(exp.warp_pt_in(cand.params))
assert_dict_equal(cand.params, cand_out)
def setup(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
param_def_wrong = {
"x": MinMaxNumericParamDef(0, 1),
"name": ["A", "B", "C"]
}
minimization = True
self.exp = Experiment(name, param_def, minimization)
assert_equal(self.exp.name, name)
assert_equal(self.exp.parameter_definitions, param_def)
assert_equal(self.exp.minimization_problem, minimization)
with assert_raises(ValueError):
Experiment("fails", False)
with assert_raises(ValueError):
Experiment("fails too", param_def_wrong)
def test_get_next_candidate(self):
exp = Experiment(
"test", {
"x": MinMaxNumericParamDef(0, 1),
"y": NominalParamDef(["A", "B", "C"])
})
opt = BayesianOptimizer(exp, {"initial_random_runs": 3})
for i in range(5):
cand = opt.get_next_candidates()[0]
assert_true(isinstance(cand, Candidate))
cand.result = 2
exp.add_finished(cand)
opt.update(exp)
cands = opt.get_next_candidates(num_candidates=3)
assert_less_equal(len(cands), 3)
def test_csv(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "A"})
exp.add_finished(cand)
string, steps_incl = exp.to_csv_results()
assert_equal(steps_incl, 1)
assert_equal(
string,
"step,name,x,cost,result,best_result\n1,A,1,None,None,None\n")
def test_add(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "A"})
cand_invalid = Candidate({"x": 1})
cand_invalid2 = Candidate({"x": 2, "name": "A"})
with assert_raises(ValueError):
exp.add_pending(cand_invalid)
with assert_raises(ValueError):
exp.add_pending(cand_invalid2)
exp.add_pending(cand)
assert cand in exp.candidates_pending
with assert_raises(ValueError):
exp.add_pending(False)
exp.add_finished(cand)
assert cand in exp.candidates_finished
with assert_raises(ValueError):
exp.add_finished(False)
cand2 = Candidate({"x": 0, "name": "B"})
exp.add_working(cand2)
assert cand2 in exp.candidates_working
with assert_raises(ValueError):
exp.add_working(False)
exp.add_pausing(cand2)
assert cand2 in exp.candidates_pending
with assert_raises(ValueError):
exp.add_pausing(False)
exp.add_working(cand2)
assert cand2 in exp.candidates_working
with assert_raises(ValueError):
exp.add_working(False)
exp.add_finished(cand2)
assert cand2 in exp.candidates_finished
with assert_raises(ValueError):
exp.add_finished(False)
def test_better_cand(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "B"})
cand2 = Candidate({"x": 0, "name": "A"})
cand_none = Candidate({"x": 0.5, "name": "C"})
cand.result = 1
cand2.result = 0
assert_true(exp.better_cand(cand2, cand))
assert_true(exp.better_cand(cand2, cand_none))
exp.minimization_problem = False
assert_true(exp.better_cand(cand, cand2))
assert_false(exp.better_cand(cand2, cand))
def test_PoI(self):
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
opt = BayesianOptimizer(exp, {
"initial_random_runs": 3,
"acquisition": ProbabilityOfImprovement
})
assert_true(
isinstance(opt.acquisition_function, ProbabilityOfImprovement))
for i in range(3):
cands = opt.get_next_candidates(2)
cand_one = cands[0]
cand_two = cands[1]
assert_true(isinstance(cand_one, Candidate))
cand_one.result = 2
exp.add_finished(cand_one)
opt.update(exp)
assert_true(isinstance(cand_two, Candidate))
cand_two.result = 2
exp.add_finished(cand_two)
opt.update(exp)
cands = opt.get_next_candidates(num_candidates=3)
assert_equal(len(cands), 3)
def test_EI(self):
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
opt = BayesianOptimizer(exp, {
"initial_random_runs": 3,
"max_searcher": "LBFGSB"
})
for i in range(3):
cands = opt.get_next_candidates(2)
cand_one = cands[0]
cand_two = cands[1]
assert_true(isinstance(cand_one, Candidate))
cand_one.result = 2
exp.add_finished(cand_one)
opt.update(exp)
assert_true(isinstance(cand_two, Candidate))
cand_two.result = 2
exp.add_finished(cand_two)
opt.update(exp)
cands = opt.get_next_candidates(num_candidates=5)
assert_equal(len(cands), 5)
opt.acquisition_function.params["multi_searcher"] = "random_best"
cands = opt.get_next_candidates(2)
def test_init(self):
#test initialization
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)}, NominalParamDef(["A", "B", "C"]))
opt = RandomSearch(exp)
