def __init__(self, name, optimizer, param_defs, experiment=None, optimizer_arguments=None,
minimization=True, write_directory_base="/tmp/APSIS_WRITING",
experiment_directory_base=None, csv_write_frequency=1):
"""
Initializes the BasicExperimentAssistant.
Parameters
----------
name : string
The name of the experiment. This does not have to be unique, but is
for human orientation.
optimizer : Optimizer instance or string
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
Alternatively, it can be a string corresponding to the optimizer,
as defined by apsis.utilities.optimizer_utils.
param_defs : dict of ParamDef.
This is the parameter space defining the experiment.
experiment : Experiment
Preinitialize this assistant with an existing experiment.
optimizer_arguments=None : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
minimization=True : bool
Whether the problem is one of minimization or maximization.
write_directory_base : string, optional
The global base directory for all writing. Will only be used
for creation of experiment_directory_base if this is not given.
experiment_directory_base : string or None, optional
The directory to write all the results to. If not
given a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int, optional
States how often the csv file should be written to.
If set to 0 no results will be written.
"""
self.logger = get_logger(self)
self.logger.info("Initializing experiment assistant.")
self.optimizer = optimizer
self.optimizer_arguments = optimizer_arguments
if experiment is None:
self.experiment = Experiment(name, param_defs, minimization)
else:
self.experiment = experiment
self.csv_write_frequency = csv_write_frequency
if self.csv_write_frequency != 0:
self.write_directory_base = write_directory_base
if experiment_directory_base is not None:
self.experiment_directory_base = experiment_directory_base
ensure_directory_exists(self.experiment_directory_base)
else:
self._create_experiment_directory()
self.logger.info("Experiment assistant successfully initialized.")
def test_get_next_candidate(self):
opt = RandomSearch({"initial_random_runs": 3})
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)}, NominalParamDef(["A", "B", "C"]))
for i in range(5):
cand = opt.get_next_candidates(exp)[0]
assert_true(isinstance(cand, Candidate))
cand.result = 2
exp.add_finished(cand)
cands = opt.get_next_candidates(exp, num_candidates=3)
assert_equal(len(cands), 3)
def test_EI(self):
opt = SimpleBayesianOptimizer({"initial_random_runs": 3})
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
for i in range(10):
cand = opt.get_next_candidates(exp)[0]
assert_true(isinstance(cand, Candidate))
cand.result = 2
exp.add_finished(cand)
cands = opt.get_next_candidates(exp, num_candidates=3)
assert_equal(len(cands), 3)
def test_warp(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1})
cand_out = exp.warp_pt_out(exp.warp_pt_in(cand.params))
assert_dict_equal(cand.params, cand_out)
def test_PoI(self):
opt = SimpleBayesianOptimizer({"initial_random_runs": 3, "acquisition": ProbabilityOfImprovement})
assert_true(isinstance(opt.acquisition_function, ProbabilityOfImprovement))
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
for i in range(10):
cand = opt.get_next_candidates(exp)[0]
assert_true(isinstance(cand, Candidate))
cand.result = 2
exp.add_finished(cand)
cands = opt.get_next_candidates(exp, num_candidates=3)
assert_equal(len(cands), 3)
def test_warp(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1})
cand_out = exp.warp_pt_out(exp.warp_pt_in(cand.params))
assert_dict_equal(cand.params, cand_out)
def test_get_next_candidate(self):
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1),
"y": NominalParamDef(["A", "B", "C"])})
opt = BayesianOptimizer(exp, {"initial_random_runs": 3})
for i in range(5):
cand = opt.get_next_candidates()[0]
assert_true(isinstance(cand, Candidate))
cand.result = 2
exp.add_finished(cand)
opt.update(exp)
cands = opt.get_next_candidates(num_candidates=3)
assert_less_equal(len(cands), 3)
def test_csv(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "A"})
exp.add_finished(cand)
string, steps_incl = exp.to_csv_results()
assert_equal(steps_incl, 1)
assert_equal(string, "step,name,x,cost,result,best_result\n1,A,1,None,None,None\n")
def test_get_next_candidate(self):
exp = Experiment(
"test", {
"x": MinMaxNumericParamDef(0, 1),
"y": NominalParamDef(["A", "B", "C"])
})
opt = BayesianOptimizer(exp, {"initial_random_runs": 3})
for i in range(5):
cand = opt.get_next_candidates()[0]
assert_true(isinstance(cand, Candidate))
cand.result = 2
exp.add_finished(cand)
opt.update(exp)
cands = opt.get_next_candidates(num_candidates=3)
assert_less_equal(len(cands), 3)
def test_csv(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "A"})
exp.add_finished(cand)
string, steps_incl = exp.to_csv_results()
assert_equal(steps_incl, 1)
assert_equal(
string,
"step,name,x,cost,result,best_result\n1,A,1,None,None,None\n")
def test_init(self):
#test default parameters
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
opt = BayesianOptimizer(exp)
assert_equal(opt.initial_random_runs, 10)
assert_is_none(opt.acquisition_hyperparams)
assert_equal(opt.num_gp_restarts, 10)
assert_true(isinstance(opt.acquisition_function, ExpectedImprovement))
assert_dict_equal(opt.kernel_params, {})
assert_equal(opt.kernel, "matern52")
#test correct initialization
opt_arguments = {
"initial_random_runs": 5,
"acquisition_hyperparams": {},
"num_gp_restarts": 5,
"acquisition": ProbabilityOfImprovement,
"kernel_params": {},
"kernel": "matern52",
"mcmc": True,
}
opt = BayesianOptimizer(exp, opt_arguments)
assert_equal(opt.initial_random_runs, 5)
assert_dict_equal(opt.acquisition_hyperparams, {})
assert_equal(opt.num_gp_restarts, 5)
assert_true(
isinstance(opt.acquisition_function, ProbabilityOfImprovement))
assert_dict_equal(opt.kernel_params, {})
assert_equal(opt.kernel, "matern52")
def setup(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1)
}
experiment = Experiment(name="test_optimizer_experiment",
parameter_definitions=param_def)
self.optimizer = self.OptimizerStub(experiment, optimizer_params=None)
def test_check_optimizer(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1)
}
experiment = Experiment(name="test_optimizer_experiment",
parameter_definitions=param_def)
assert_is_instance(check_optimizer(RandomSearch, experiment, {"multiprocessing": "none"}), RandomSearch)
assert_is_instance(check_optimizer("RandomSearch", experiment, {"multiprocessing": "none"}), RandomSearch)
with assert_raises(ValueError):
check_optimizer("fails", experiment, {"multiprocessing": "none"})
with assert_raises(ValueError):
check_optimizer(MinMaxNumericParamDef, experiment)
queue_based = check_optimizer(RandomSearch, experiment,
{"multiprocessing": "queue"})
assert_is_instance(queue_based, QueueBasedOptimizer)
assert_equal(check_optimizer(queue_based, experiment,
{"multiprocessing": "queue"}), queue_based)
queue_based.exit()
assert_is_instance(check_optimizer(RandomSearch, experiment,
{"multiprocessing": "none"}),
RandomSearch)
with assert_raises(ValueError):
check_optimizer(RandomSearch, experiment,
{"multiprocessing": "fails"}),
time.sleep(0.1)
def test_update(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1)
}
experiment = Experiment(name="test_optimizer_experiment",
parameter_definitions=param_def)
self.optimizer.update(experiment)
def setup(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1)
}
experiment = Experiment(name="test_optimizer_experiment",
parameter_definitions=param_def)
self.optimizer = QueueBasedOptimizer(RandomSearch,
experiment)
def test_update_param_support(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"not_supported": OrdinalParamDef(["A", "B"])
}
experiment = Experiment(name="test_optimizer_experiment_crash",
parameter_definitions=param_def)
assert_raises(ValueError, self.optimizer.update, experiment)
def test_PoI(self):
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
opt = BayesianOptimizer(exp, {"initial_random_runs": 3, "acquisition": ProbabilityOfImprovement})
assert_true(isinstance(opt.acquisition_function, ProbabilityOfImprovement))
for i in range(3):
cands = opt.get_next_candidates(2)
cand_one = cands[0]
cand_two = cands[1]
assert_true(isinstance(cand_one, Candidate))
cand_one.result = 2
exp.add_finished(cand_one)
opt.update(exp)
assert_true(isinstance(cand_two, Candidate))
cand_two.result = 2
exp.add_finished(cand_two)
opt.update(exp)
cands = opt.get_next_candidates(num_candidates=3)
assert_equal(len(cands), 3)
def setup(self):
param_def = {
"x": MinMaxNumericParamDef(0, 1)
}
self.experiment = Experiment(name="test_optimizer_experiment",
parameter_definitions=param_def)
out_queue = Queue()
in_queue = Queue()
self.backend = QueueBackend(RandomSearch, self.experiment, out_queue,
in_queue)
def test_EI(self):
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
opt = BayesianOptimizer(exp, {"initial_random_runs": 3, "max_searcher": "LBFGSB"})
for i in range(3):
cands = opt.get_next_candidates(2)
cand_one = cands[0]
cand_two = cands[1]
assert_true(isinstance(cand_one, Candidate))
cand_one.result = 2
exp.add_finished(cand_one)
opt.update(exp)
assert_true(isinstance(cand_two, Candidate))
cand_two.result = 2
exp.add_finished(cand_two)
opt.update(exp)
cands = opt.get_next_candidates(num_candidates=5)
assert_equal(len(cands), 5)
opt.acquisition_function.params["multi_searcher"] = "random_best"
cands = opt.get_next_candidates(2)
def setup(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
param_def_wrong = {
"x": MinMaxNumericParamDef(0, 1),
"name": ["A", "B", "C"]
}
minimization = True
self.exp = Experiment(name, param_def, minimization)
assert_equal(self.exp.name, name)
assert_equal(self.exp.parameter_definitions, param_def)
assert_equal(self.exp.minimization_problem, minimization)
with assert_raises(ValueError):
Experiment("fails", False)
with assert_raises(ValueError):
Experiment("fails too", param_def_wrong)
def test_better_cand(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "B"})
cand2 = Candidate({"x": 0, "name": "A"})
cand_none = Candidate({"x": 0.5, "name": "C"})
cand.result = 1
cand2.result = 0
assert_true(exp.better_cand(cand2, cand))
assert_true(exp.better_cand(cand2, cand_none))
exp.minimization_problem = False
assert_true(exp.better_cand(cand, cand2))
assert_false(exp.better_cand(cand2, cand))
def setup(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
param_def_wrong = {
"x": MinMaxNumericParamDef(0, 1),
"name": ["A", "B", "C"]
}
minimization = True
self.exp = Experiment(name, param_def, minimization)
assert_equal(self.exp.name, name)
assert_equal(self.exp.parameter_definitions, param_def)
assert_equal(self.exp.minimization_problem, minimization)
with assert_raises(ValueError):
Experiment("fails", False)
with assert_raises(ValueError):
Experiment("fails too", param_def_wrong)
def test_better_cand(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "B"})
cand2 = Candidate({"x": 0, "name": "A"})
cand_none = Candidate({"x": 0.5, "name": "C"})
cand.result = 1
cand2.result = 0
assert_true(exp.better_cand(cand2, cand))
assert_true(exp.better_cand(cand2, cand_none))
exp.minimization_problem = False
assert_true(exp.better_cand(cand, cand2))
assert_false(exp.better_cand(cand2, cand))
def test_PoI(self):
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
opt = BayesianOptimizer(exp, {
"initial_random_runs": 3,
"acquisition": ProbabilityOfImprovement
})
assert_true(
isinstance(opt.acquisition_function, ProbabilityOfImprovement))
for i in range(3):
cands = opt.get_next_candidates(2)
cand_one = cands[0]
cand_two = cands[1]
assert_true(isinstance(cand_one, Candidate))
cand_one.result = 2
exp.add_finished(cand_one)
opt.update(exp)
assert_true(isinstance(cand_two, Candidate))
cand_two.result = 2
exp.add_finished(cand_two)
opt.update(exp)
cands = opt.get_next_candidates(num_candidates=3)
assert_equal(len(cands), 3)
def test_EI(self):
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
opt = BayesianOptimizer(exp, {
"initial_random_runs": 3,
"max_searcher": "LBFGSB"
})
for i in range(3):
cands = opt.get_next_candidates(2)
cand_one = cands[0]
cand_two = cands[1]
assert_true(isinstance(cand_one, Candidate))
cand_one.result = 2
exp.add_finished(cand_one)
opt.update(exp)
assert_true(isinstance(cand_two, Candidate))
cand_two.result = 2
exp.add_finished(cand_two)
opt.update(exp)
cands = opt.get_next_candidates(num_candidates=5)
assert_equal(len(cands), 5)
opt.acquisition_function.params["multi_searcher"] = "random_best"
cands = opt.get_next_candidates(2)
class TestExperiment(object):
exp = None
def setup(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
param_def_wrong = {
"x": MinMaxNumericParamDef(0, 1),
"name": ["A", "B", "C"]
}
minimization = True
self.exp = Experiment(name, param_def, minimization)
assert_equal(self.exp.name, name)
assert_equal(self.exp.parameter_definitions, param_def)
assert_equal(self.exp.minimization_problem, minimization)
with assert_raises(ValueError):
Experiment("fails", False)
with assert_raises(ValueError):
Experiment("fails too", param_def_wrong)
def test_add(self):
cand = Candidate({"x": 1, "name": "A"})
cand_invalid = Candidate({"x": 1})
cand_invalid2 = Candidate({"x": 2, "name": "A"})
with assert_raises(ValueError):
self.exp.add_pending(cand_invalid)
with assert_raises(ValueError):
self.exp.add_pending(cand_invalid2)
self.exp.add_pending(cand)
assert cand in self.exp.candidates_pending
with assert_raises(ValueError):
self.exp.add_pending(False)
self.exp.add_finished(cand)
assert cand in self.exp.candidates_finished
with assert_raises(ValueError):
self.exp.add_finished(False)
cand2 = Candidate({"x": 0, "name": "B"})
self.exp.add_working(cand2)
assert cand2 in self.exp.candidates_working
with assert_raises(ValueError):
self.exp.add_working(False)
self.exp.add_pausing(cand2)
assert cand2 in self.exp.candidates_pending
with assert_raises(ValueError):
self.exp.add_pausing(False)
self.exp.add_working(cand2)
assert cand2 in self.exp.candidates_working
with assert_raises(ValueError):
self.exp.add_working(False)
self.exp.add_finished(cand2)
assert cand2 in self.exp.candidates_finished
with assert_raises(ValueError):
self.exp.add_finished(False)
def test_better_cand(self):
cand = Candidate({"x": 1, "name": "B"})
cand2 = Candidate({"x": 0, "name": "A"})
cand_none = Candidate({"x": 0.5, "name": "C"})
cand_invalid = Candidate({"x": 0.5, "name": "D"})
cand.result = 1
cand2.result = 0
assert_true(self.exp.better_cand(cand2, cand))
assert_true(self.exp.better_cand(cand2, cand_none))
self.exp.minimization_problem = False
assert_true(self.exp.better_cand(cand, cand2))
assert_false(self.exp.better_cand(cand2, cand))
assert_true(self.exp.better_cand(cand, None))
assert_false(self.exp.better_cand(None, cand))
assert_false(self.exp.better_cand(None, None))
with assert_raises(ValueError):
self.exp.better_cand(cand, cand_invalid)
with assert_raises(ValueError):
self.exp.better_cand(cand_invalid, cand)
with assert_raises(ValueError):
self.exp.better_cand("fails", cand)
with assert_raises(ValueError):
self.exp.better_cand(cand, "fails")
def test_warp(self):
cand = Candidate({"x": 1})
cand_out = self.exp.warp_pt_out(self.exp.warp_pt_in(cand.params))
assert_dict_equal(cand.params, cand_out)
def test_csv(self):
cand = Candidate({"x": 1, "name": "A"})
self.exp.add_finished(cand)
string, steps_incl = self.exp.to_csv_results()
assert_equal(steps_incl, 1)
assert_equal(string, "step,id,name,x,cost,result,best_result\n1,%s,A,1,None,None,None\n"%cand.id)
def test_to_dict(self):
cand = Candidate({"x": 1, "name": "A"})
self.exp.add_finished(cand)
self.exp.to_dict()
def test_check_param_dict(self):
param_dict = {"x": 1}
assert_false(self.exp._check_param_dict(param_dict))
param_dict = {"x": 1,
"name": "D"}
assert_false(self.exp._check_param_dict(param_dict))
param_dict = {"x": 1,
"name": "A"}
assert_true(self.exp._check_param_dict(param_dict))
def test_add(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "A"})
cand_invalid = Candidate({"x": 1})
cand_invalid2 = Candidate({"x": 2, "name": "A"})
with assert_raises(ValueError):
exp.add_pending(cand_invalid)
with assert_raises(ValueError):
exp.add_pending(cand_invalid2)
exp.add_pending(cand)
assert cand in exp.candidates_pending
with assert_raises(ValueError):
exp.add_pending(False)
exp.add_finished(cand)
assert cand in exp.candidates_finished
with assert_raises(ValueError):
exp.add_finished(False)
cand2 = Candidate({"x": 0, "name": "B"})
exp.add_working(cand2)
assert cand2 in exp.candidates_working
with assert_raises(ValueError):
exp.add_working(False)
exp.add_pausing(cand2)
assert cand2 in exp.candidates_pending
with assert_raises(ValueError):
exp.add_pausing(False)
exp.add_working(cand2)
assert cand2 in exp.candidates_working
with assert_raises(ValueError):
exp.add_working(False)
exp.add_finished(cand2)
assert cand2 in exp.candidates_finished
with assert_raises(ValueError):
exp.add_finished(False)
def test_add(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "A"})
cand_invalid = Candidate({"x": 1})
cand_invalid2 = Candidate({"x": 2, "name": "A"})
with assert_raises(ValueError):
exp.add_pending(cand_invalid)
with assert_raises(ValueError):
exp.add_pending(cand_invalid2)
exp.add_pending(cand)
assert cand in exp.candidates_pending
with assert_raises(ValueError):
exp.add_pending(False)
exp.add_finished(cand)
assert cand in exp.candidates_finished
with assert_raises(ValueError):
exp.add_finished(False)
cand2 = Candidate({"x": 0, "name": "B"})
exp.add_working(cand2)
assert cand2 in exp.candidates_working
with assert_raises(ValueError):
exp.add_working(False)
exp.add_pausing(cand2)
assert cand2 in exp.candidates_pending
with assert_raises(ValueError):
exp.add_pausing(False)
exp.add_working(cand2)
assert cand2 in exp.candidates_working
with assert_raises(ValueError):
exp.add_working(False)
exp.add_finished(cand2)
assert cand2 in exp.candidates_finished
with assert_raises(ValueError):
exp.add_finished(False)
class BasicExperimentAssistant(object):
"""
This ExperimentAssistant assists with executing experiments.
It provides methods for getting candidates to evaluate, returning the
evaluated Candidate and administrates the optimizer.
Attributes
----------
optimizer : Optimizer
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
optimizer_arguments : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
experiment : Experiment
The experiment this assistant assists with.
write_directory_base : string or None
The directory to write all results to. If not
given, a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int
States how often the csv file should be written to.
If set to 0 no results will be written.
logger : logging.logger
The logger for this class.
"""
AVAILABLE_STATUS = ["finished", "pausing", "working"]
optimizer = None
optimizer_arguments = None
experiment = None
write_directory_base = None
experiment_directory_base = None
csv_write_frequency = None
csv_steps_written = 0
logger = None
def __init__(
self,
name,
optimizer,
param_defs,
experiment=None,
optimizer_arguments=None,
minimization=True,
write_directory_base="/tmp/APSIS_WRITING",
experiment_directory_base=None,
csv_write_frequency=1,
):
"""
Initializes the BasicExperimentAssistant.
Parameters
----------
name : string
The name of the experiment. This does not have to be unique, but is
for human orientation.
optimizer : Optimizer instance or string
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
Alternatively, it can be a string corresponding to the optimizer,
as defined by apsis.utilities.optimizer_utils.
param_defs : dict of ParamDef.
This is the parameter space defining the experiment.
experiment : Experiment
Preinitialize this assistant with an existing experiment.
optimizer_arguments=None : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
minimization=True : bool
Whether the problem is one of minimization or maximization.
write_directory_base : string, optional
The global base directory for all writing. Will only be used
for creation of experiment_directory_base if this is not given.
experiment_directory_base : string or None, optional
The directory to write all the results to. If not
given a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int, optional
States how often the csv file should be written to.
If set to 0 no results will be written.
"""
self.logger = get_logger(self)
self.logger.info("Initializing experiment assistant.")
self.optimizer = optimizer
self.optimizer_arguments = optimizer_arguments
if experiment is None:
self.experiment = Experiment(name, param_defs, minimization)
else:
self.experiment = experiment
self.csv_write_frequency = csv_write_frequency
if self.csv_write_frequency != 0:
self.write_directory_base = write_directory_base
if experiment_directory_base is not None:
self.experiment_directory_base = experiment_directory_base
ensure_directory_exists(self.experiment_directory_base)
else:
self._create_experiment_directory()
self.logger.info("Experiment assistant successfully initialized.")
def get_next_candidate(self):
"""
Returns the Candidate next to evaluate.
Returns
-------
next_candidate : Candidate or None
The Candidate object that should be evaluated next. May be None.
"""
self.logger.info("Returning next candidate.")
self.optimizer = check_optimizer(self.optimizer, optimizer_arguments=self.optimizer_arguments)
if not self.experiment.candidates_pending:
self.experiment.candidates_pending.extend(self.optimizer.get_next_candidates(self.experiment))
next_candidate = self.experiment.candidates_pending.pop()
self.logger.info("next candidate found: %s" % next_candidate)
return next_candidate
def update(self, candidate, status="finished"):
"""
Updates the experiment_assistant with the status of an experiment
evaluation.
Parameters
----------
candidate : Candidate
The Candidate object whose status is updated.
status : {"finished", "pausing", "working"}
A string defining the status change. Can be one of the following:
- finished: The Candidate is now finished.
- pausing: The evaluation of Candidate has been paused and can be
resumed by another worker.
- working: The Candidate is now being worked on by a worker.
"""
if status not in self.AVAILABLE_STATUS:
message = "status not in %s but %s." % (str(self.AVAILABLE_STATUS), str(status))
self.logger.error(message)
raise ValueError(message)
if not isinstance(candidate, Candidate):
message = "candidate %s not a Candidate instance." % str(candidate)
self.logger.error(message)
raise ValueError(message)
self.logger.info(
"Got new %s of candidate %s with parameters %s"
" and result %s" % (status, candidate, candidate.params, candidate.result)
)
if status == "finished":
self.experiment.add_finished(candidate)
# Also delete all pending candidates from the experiment - we have
# new data available.
self.experiment.candidates_pending = []
# invoke the writing to files
step = len(self.experiment.candidates_finished)
if self.csv_write_frequency != 0 and step != 0 and step % self.csv_write_frequency == 0:
self._append_to_detailed_csv()
elif status == "pausing":
self.experiment.add_pausing(candidate)
elif status == "working":
self.experiment.add_working(candidate)
def get_best_candidate(self):
"""
Returns the best candidate to date.
Returns
-------
best_candidate : candidate or None
Returns a candidate if there is a best one (which corresponds to
at least one candidate evaluated) or None if none exists.
"""
return self.experiment.best_candidate
def _append_to_detailed_csv(self):
if len(self.experiment.candidates_finished) <= self.csv_steps_written:
return
# create file and header if
wHeader = False
if self.csv_steps_written == 0:
# set use header
wHeader = True
csv_string, steps_included = self.experiment.to_csv_results(wHeader=wHeader, fromIndex=self.csv_steps_written)
# write
filename = os.path.join(self.experiment_directory_base, self.experiment.name + "_results.csv")
with open(filename, "a+") as detailed_file:
detailed_file.write(csv_string)
self.csv_steps_written += steps_included
def _create_experiment_directory(self):
global_start_date = time.time()
date_name = datetime.datetime.utcfromtimestamp(global_start_date).strftime("%Y-%m-%d_%H:%M:%S")
self.experiment_directory_base = os.path.join(self.write_directory_base, self.experiment.name + "_" + date_name)
ensure_directory_exists(self.experiment_directory_base)
def __init__(
self,
name,
optimizer,
param_defs,
experiment=None,
optimizer_arguments=None,
minimization=True,
write_directory_base="/tmp/APSIS_WRITING",
experiment_directory_base=None,
csv_write_frequency=1,
):
"""
Initializes the BasicExperimentAssistant.
Parameters
----------
name : string
The name of the experiment. This does not have to be unique, but is
for human orientation.
optimizer : Optimizer instance or string
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
Alternatively, it can be a string corresponding to the optimizer,
as defined by apsis.utilities.optimizer_utils.
param_defs : dict of ParamDef.
This is the parameter space defining the experiment.
experiment : Experiment
Preinitialize this assistant with an existing experiment.
optimizer_arguments=None : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
minimization=True : bool
Whether the problem is one of minimization or maximization.
write_directory_base : string, optional
The global base directory for all writing. Will only be used
for creation of experiment_directory_base if this is not given.
experiment_directory_base : string or None, optional
The directory to write all the results to. If not
given a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int, optional
States how often the csv file should be written to.
If set to 0 no results will be written.
"""
self.logger = get_logger(self)
self.logger.info("Initializing experiment assistant.")
self.optimizer = optimizer
self.optimizer_arguments = optimizer_arguments
if experiment is None:
self.experiment = Experiment(name, param_defs, minimization)
else:
self.experiment = experiment
self.csv_write_frequency = csv_write_frequency
if self.csv_write_frequency != 0:
self.write_directory_base = write_directory_base
if experiment_directory_base is not None:
self.experiment_directory_base = experiment_directory_base
ensure_directory_exists(self.experiment_directory_base)
else:
self._create_experiment_directory()
self.logger.info("Experiment assistant successfully initialized.")
class BasicExperimentAssistant(object):
"""
This ExperimentAssistant assists with executing experiments.
It provides methods for getting candidates to evaluate, returning the
evaluated Candidate and administrates the optimizer.
Attributes
----------
optimizer : Optimizer
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
optimizer_arguments : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
experiment : Experiment
The experiment this assistant assists with.
write_directory_base : string or None
The directory to write all results to. If not
given, a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int
States how often the csv file should be written to.
If set to 0 no results will be written.
logger : logging.logger
The logger for this class.
"""
AVAILABLE_STATUS = ["finished", "pausing", "working"]
optimizer = None
optimizer_arguments = None
experiment = None
write_directory_base = None
experiment_directory_base = None
csv_write_frequency = None
csv_steps_written = 0
logger = None
def __init__(self, name, optimizer, param_defs, experiment=None, optimizer_arguments=None,
minimization=True, write_directory_base="/tmp/APSIS_WRITING",
experiment_directory_base=None, csv_write_frequency=1):
"""
Initializes the BasicExperimentAssistant.
Parameters
----------
name : string
The name of the experiment. This does not have to be unique, but is
for human orientation.
optimizer : Optimizer instance or string
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
Alternatively, it can be a string corresponding to the optimizer,
as defined by apsis.utilities.optimizer_utils.
param_defs : dict of ParamDef.
This is the parameter space defining the experiment.
experiment : Experiment
Preinitialize this assistant with an existing experiment.
optimizer_arguments=None : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
minimization=True : bool
Whether the problem is one of minimization or maximization.
write_directory_base : string, optional
The global base directory for all writing. Will only be used
for creation of experiment_directory_base if this is not given.
experiment_directory_base : string or None, optional
The directory to write all the results to. If not
given a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int, optional
States how often the csv file should be written to.
If set to 0 no results will be written.
"""
self.logger = get_logger(self)
self.logger.info("Initializing experiment assistant.")
self.optimizer = optimizer
self.optimizer_arguments = optimizer_arguments
if experiment is None:
self.experiment = Experiment(name, param_defs, minimization)
else:
self.experiment = experiment
self.csv_write_frequency = csv_write_frequency
if self.csv_write_frequency != 0:
self.write_directory_base = write_directory_base
if experiment_directory_base is not None:
self.experiment_directory_base = experiment_directory_base
ensure_directory_exists(self.experiment_directory_base)
else:
self._create_experiment_directory()
self.logger.info("Experiment assistant successfully initialized.")
def get_next_candidate(self):
"""
Returns the Candidate next to evaluate.
Returns
-------
next_candidate : Candidate or None
The Candidate object that should be evaluated next. May be None.
"""
self.logger.info("Returning next candidate.")
self.optimizer = check_optimizer(self.optimizer,
optimizer_arguments=self.optimizer_arguments)
if not self.experiment.candidates_pending:
self.experiment.candidates_pending.extend(
self.optimizer.get_next_candidates(self.experiment))
next_candidate = self.experiment.candidates_pending.pop()
self.logger.info("next candidate found: %s" %next_candidate)
return next_candidate
def update(self, candidate, status="finished"):
"""
Updates the experiment_assistant with the status of an experiment
evaluation.
Parameters
----------
candidate : Candidate
The Candidate object whose status is updated.
status : {"finished", "pausing", "working"}
A string defining the status change. Can be one of the following:
- finished: The Candidate is now finished.
- pausing: The evaluation of Candidate has been paused and can be
resumed by another worker.
- working: The Candidate is now being worked on by a worker.
"""
if status not in self.AVAILABLE_STATUS:
message = ("status not in %s but %s."
%(str(self.AVAILABLE_STATUS), str(status)))
self.logger.error(message)
raise ValueError(message)
if not isinstance(candidate, Candidate):
message = ("candidate %s not a Candidate instance."
%str(candidate))
self.logger.error(message)
raise ValueError(message)
self.logger.info("Got new %s of candidate %s with parameters %s"
" and result %s" %(status, candidate, candidate.params,
candidate.result))
if status == "finished":
self.experiment.add_finished(candidate)
#Also delete all pending candidates from the experiment - we have
#new data available.
self.experiment.candidates_pending = []
#invoke the writing to files
step = len(self.experiment.candidates_finished)
if self.csv_write_frequency != 0 and step != 0 \
and step % self.csv_write_frequency == 0:
self._append_to_detailed_csv()
elif status == "pausing":
self.experiment.add_pausing(candidate)
elif status == "working":
self.experiment.add_working(candidate)
def get_best_candidate(self):
"""
Returns the best candidate to date.
Returns
-------
best_candidate : candidate or None
Returns a candidate if there is a best one (which corresponds to
at least one candidate evaluated) or None if none exists.
"""
return self.experiment.best_candidate
def _append_to_detailed_csv(self):
if len(self.experiment.candidates_finished) <= self.csv_steps_written:
return
#create file and header if
wHeader = False
if self.csv_steps_written == 0:
#set use header
wHeader = True
csv_string, steps_included = self.experiment.to_csv_results(wHeader=wHeader,
fromIndex=self.csv_steps_written)
#write
filename = os.path.join(self.experiment_directory_base,
self.experiment.name + "_results.csv")
with open(filename, 'a+') as detailed_file:
detailed_file.write(csv_string)
self.csv_steps_written += steps_included
def _create_experiment_directory(self):
global_start_date = time.time()
date_name = datetime.datetime.utcfromtimestamp(
global_start_date).strftime("%Y-%m-%d_%H:%M:%S")
self.experiment_directory_base = os.path.join(self.write_directory_base,
self.experiment.name + "_" + date_name)
ensure_directory_exists(self.experiment_directory_base)
class TestExperiment(object):
exp = None
def setup(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
param_def_wrong = {
"x": MinMaxNumericParamDef(0, 1),
"name": ["A", "B", "C"]
}
minimization = True
self.exp = Experiment(name, param_def, minimization)
assert_equal(self.exp.name, name)
assert_equal(self.exp.parameter_definitions, param_def)
assert_equal(self.exp.minimization_problem, minimization)
with assert_raises(ValueError):
Experiment("fails", False)
with assert_raises(ValueError):
Experiment("fails too", param_def_wrong)
def test_add(self):
cand = Candidate({"x": 1, "name": "A"})
cand_invalid = Candidate({"x": 1})
cand_invalid2 = Candidate({"x": 2, "name": "A"})
with assert_raises(ValueError):
self.exp.add_pending(cand_invalid)
with assert_raises(ValueError):
self.exp.add_pending(cand_invalid2)
self.exp.add_pending(cand)
assert cand in self.exp.candidates_pending
with assert_raises(ValueError):
self.exp.add_pending(False)
self.exp.add_finished(cand)
assert cand in self.exp.candidates_finished
with assert_raises(ValueError):
self.exp.add_finished(False)
cand2 = Candidate({"x": 0, "name": "B"})
self.exp.add_working(cand2)
assert cand2 in self.exp.candidates_working
with assert_raises(ValueError):
self.exp.add_working(False)
self.exp.add_pausing(cand2)
assert cand2 in self.exp.candidates_pending
with assert_raises(ValueError):
self.exp.add_pausing(False)
self.exp.add_working(cand2)
assert cand2 in self.exp.candidates_working
with assert_raises(ValueError):
self.exp.add_working(False)
self.exp.add_finished(cand2)
assert cand2 in self.exp.candidates_finished
with assert_raises(ValueError):
self.exp.add_finished(False)
def test_better_cand(self):
cand = Candidate({"x": 1, "name": "B"})
cand2 = Candidate({"x": 0, "name": "A"})
cand_none = Candidate({"x": 0.5, "name": "C"})
cand_invalid = Candidate({"x": 0.5, "name": "D"})
cand.result = 1
cand2.result = 0
assert_true(self.exp.better_cand(cand2, cand))
assert_true(self.exp.better_cand(cand2, cand_none))
self.exp.minimization_problem = False
assert_true(self.exp.better_cand(cand, cand2))
assert_false(self.exp.better_cand(cand2, cand))
assert_true(self.exp.better_cand(cand, None))
assert_false(self.exp.better_cand(None, cand))
assert_false(self.exp.better_cand(None, None))
with assert_raises(ValueError):
self.exp.better_cand(cand, cand_invalid)
with assert_raises(ValueError):
self.exp.better_cand(cand_invalid, cand)
with assert_raises(ValueError):
self.exp.better_cand("fails", cand)
with assert_raises(ValueError):
self.exp.better_cand(cand, "fails")
def test_warp(self):
cand = Candidate({"x": 1})
cand_out = self.exp.warp_pt_out(self.exp.warp_pt_in(cand.params))
assert_dict_equal(cand.params, cand_out)
def test_to_dict(self):
cand = Candidate({"x": 1, "name": "A"})
self.exp.add_finished(cand)
self.exp.to_dict()
def test_check_param_dict(self):
param_dict = {"x": 1}
assert_false(self.exp._check_param_dict(param_dict))
param_dict = {"x": 1,
"name": "D"}
assert_false(self.exp._check_param_dict(param_dict))
param_dict = {"x": 1,
"name": "A"}
assert_true(self.exp._check_param_dict(param_dict))
def test_init(self):
#test initialization
exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)}, NominalParamDef(["A", "B", "C"]))
opt = RandomSearch(exp)
