def test_add(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "A"})
cand_invalid = Candidate({"x": 1})
cand_invalid2 = Candidate({"x": 2, "name": "A"})
with assert_raises(ValueError):
exp.add_pending(cand_invalid)
with assert_raises(ValueError):
exp.add_pending(cand_invalid2)
exp.add_pending(cand)
assert cand in exp.candidates_pending
with assert_raises(ValueError):
exp.add_pending(False)
exp.add_finished(cand)
assert cand in exp.candidates_finished
with assert_raises(ValueError):
exp.add_finished(False)
cand2 = Candidate({"x": 0, "name": "B"})
exp.add_working(cand2)
assert cand2 in exp.candidates_working
with assert_raises(ValueError):
exp.add_working(False)
exp.add_pausing(cand2)
assert cand2 in exp.candidates_pending
with assert_raises(ValueError):
exp.add_pausing(False)
exp.add_working(cand2)
assert cand2 in exp.candidates_working
with assert_raises(ValueError):
exp.add_working(False)
exp.add_finished(cand2)
assert cand2 in exp.candidates_finished
with assert_raises(ValueError):
exp.add_finished(False)
def test_add(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
minimization = True
exp = Experiment(name, param_def, minimization)
cand = Candidate({"x": 1, "name": "A"})
cand_invalid = Candidate({"x": 1})
cand_invalid2 = Candidate({"x": 2, "name": "A"})
with assert_raises(ValueError):
exp.add_pending(cand_invalid)
with assert_raises(ValueError):
exp.add_pending(cand_invalid2)
exp.add_pending(cand)
assert cand in exp.candidates_pending
with assert_raises(ValueError):
exp.add_pending(False)
exp.add_finished(cand)
assert cand in exp.candidates_finished
with assert_raises(ValueError):
exp.add_finished(False)
cand2 = Candidate({"x": 0, "name": "B"})
exp.add_working(cand2)
assert cand2 in exp.candidates_working
with assert_raises(ValueError):
exp.add_working(False)
exp.add_pausing(cand2)
assert cand2 in exp.candidates_pending
with assert_raises(ValueError):
exp.add_pausing(False)
exp.add_working(cand2)
assert cand2 in exp.candidates_working
with assert_raises(ValueError):
exp.add_working(False)
exp.add_finished(cand2)
assert cand2 in exp.candidates_finished
with assert_raises(ValueError):
exp.add_finished(False)
class TestExperiment(object):
exp = None
def setup(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
param_def_wrong = {
"x": MinMaxNumericParamDef(0, 1),
"name": ["A", "B", "C"]
}
minimization = True
self.exp = Experiment(name, param_def, minimization)
assert_equal(self.exp.name, name)
assert_equal(self.exp.parameter_definitions, param_def)
assert_equal(self.exp.minimization_problem, minimization)
with assert_raises(ValueError):
Experiment("fails", False)
with assert_raises(ValueError):
Experiment("fails too", param_def_wrong)
def test_add(self):
cand = Candidate({"x": 1, "name": "A"})
cand_invalid = Candidate({"x": 1})
cand_invalid2 = Candidate({"x": 2, "name": "A"})
with assert_raises(ValueError):
self.exp.add_pending(cand_invalid)
with assert_raises(ValueError):
self.exp.add_pending(cand_invalid2)
self.exp.add_pending(cand)
assert cand in self.exp.candidates_pending
with assert_raises(ValueError):
self.exp.add_pending(False)
self.exp.add_finished(cand)
assert cand in self.exp.candidates_finished
with assert_raises(ValueError):
self.exp.add_finished(False)
cand2 = Candidate({"x": 0, "name": "B"})
self.exp.add_working(cand2)
assert cand2 in self.exp.candidates_working
with assert_raises(ValueError):
self.exp.add_working(False)
self.exp.add_pausing(cand2)
assert cand2 in self.exp.candidates_pending
with assert_raises(ValueError):
self.exp.add_pausing(False)
self.exp.add_working(cand2)
assert cand2 in self.exp.candidates_working
with assert_raises(ValueError):
self.exp.add_working(False)
self.exp.add_finished(cand2)
assert cand2 in self.exp.candidates_finished
with assert_raises(ValueError):
self.exp.add_finished(False)
def test_better_cand(self):
cand = Candidate({"x": 1, "name": "B"})
cand2 = Candidate({"x": 0, "name": "A"})
cand_none = Candidate({"x": 0.5, "name": "C"})
cand_invalid = Candidate({"x": 0.5, "name": "D"})
cand.result = 1
cand2.result = 0
assert_true(self.exp.better_cand(cand2, cand))
assert_true(self.exp.better_cand(cand2, cand_none))
self.exp.minimization_problem = False
assert_true(self.exp.better_cand(cand, cand2))
assert_false(self.exp.better_cand(cand2, cand))
assert_true(self.exp.better_cand(cand, None))
assert_false(self.exp.better_cand(None, cand))
assert_false(self.exp.better_cand(None, None))
with assert_raises(ValueError):
self.exp.better_cand(cand, cand_invalid)
with assert_raises(ValueError):
self.exp.better_cand(cand_invalid, cand)
with assert_raises(ValueError):
self.exp.better_cand("fails", cand)
with assert_raises(ValueError):
self.exp.better_cand(cand, "fails")
def test_warp(self):
cand = Candidate({"x": 1})
cand_out = self.exp.warp_pt_out(self.exp.warp_pt_in(cand.params))
assert_dict_equal(cand.params, cand_out)
def test_csv(self):
cand = Candidate({"x": 1, "name": "A"})
self.exp.add_finished(cand)
string, steps_incl = self.exp.to_csv_results()
assert_equal(steps_incl, 1)
assert_equal(string, "step,id,name,x,cost,result,best_result\n1,%s,A,1,None,None,None\n"%cand.id)
def test_to_dict(self):
cand = Candidate({"x": 1, "name": "A"})
self.exp.add_finished(cand)
self.exp.to_dict()
def test_check_param_dict(self):
param_dict = {"x": 1}
assert_false(self.exp._check_param_dict(param_dict))
param_dict = {"x": 1,
"name": "D"}
assert_false(self.exp._check_param_dict(param_dict))
param_dict = {"x": 1,
"name": "A"}
assert_true(self.exp._check_param_dict(param_dict))
class TestExperiment(object):
exp = None
def setup(self):
name = "test_experiment"
param_def = {
"x": MinMaxNumericParamDef(0, 1),
"name": NominalParamDef(["A", "B", "C"])
}
param_def_wrong = {
"x": MinMaxNumericParamDef(0, 1),
"name": ["A", "B", "C"]
}
minimization = True
self.exp = Experiment(name, param_def, minimization)
assert_equal(self.exp.name, name)
assert_equal(self.exp.parameter_definitions, param_def)
assert_equal(self.exp.minimization_problem, minimization)
with assert_raises(ValueError):
Experiment("fails", False)
with assert_raises(ValueError):
Experiment("fails too", param_def_wrong)
def test_add(self):
cand = Candidate({"x": 1, "name": "A"})
cand_invalid = Candidate({"x": 1})
cand_invalid2 = Candidate({"x": 2, "name": "A"})
with assert_raises(ValueError):
self.exp.add_pending(cand_invalid)
with assert_raises(ValueError):
self.exp.add_pending(cand_invalid2)
self.exp.add_pending(cand)
assert cand in self.exp.candidates_pending
with assert_raises(ValueError):
self.exp.add_pending(False)
self.exp.add_finished(cand)
assert cand in self.exp.candidates_finished
with assert_raises(ValueError):
self.exp.add_finished(False)
cand2 = Candidate({"x": 0, "name": "B"})
self.exp.add_working(cand2)
assert cand2 in self.exp.candidates_working
with assert_raises(ValueError):
self.exp.add_working(False)
self.exp.add_pausing(cand2)
assert cand2 in self.exp.candidates_pending
with assert_raises(ValueError):
self.exp.add_pausing(False)
self.exp.add_working(cand2)
assert cand2 in self.exp.candidates_working
with assert_raises(ValueError):
self.exp.add_working(False)
self.exp.add_finished(cand2)
assert cand2 in self.exp.candidates_finished
with assert_raises(ValueError):
self.exp.add_finished(False)
def test_better_cand(self):
cand = Candidate({"x": 1, "name": "B"})
cand2 = Candidate({"x": 0, "name": "A"})
cand_none = Candidate({"x": 0.5, "name": "C"})
cand_invalid = Candidate({"x": 0.5, "name": "D"})
cand.result = 1
cand2.result = 0
assert_true(self.exp.better_cand(cand2, cand))
assert_true(self.exp.better_cand(cand2, cand_none))
self.exp.minimization_problem = False
assert_true(self.exp.better_cand(cand, cand2))
assert_false(self.exp.better_cand(cand2, cand))
assert_true(self.exp.better_cand(cand, None))
assert_false(self.exp.better_cand(None, cand))
assert_false(self.exp.better_cand(None, None))
with assert_raises(ValueError):
self.exp.better_cand(cand, cand_invalid)
with assert_raises(ValueError):
self.exp.better_cand(cand_invalid, cand)
with assert_raises(ValueError):
self.exp.better_cand("fails", cand)
with assert_raises(ValueError):
self.exp.better_cand(cand, "fails")
def test_warp(self):
cand = Candidate({"x": 1})
cand_out = self.exp.warp_pt_out(self.exp.warp_pt_in(cand.params))
assert_dict_equal(cand.params, cand_out)
def test_to_dict(self):
cand = Candidate({"x": 1, "name": "A"})
self.exp.add_finished(cand)
self.exp.to_dict()
def test_check_param_dict(self):
param_dict = {"x": 1}
assert_false(self.exp._check_param_dict(param_dict))
param_dict = {"x": 1,
"name": "D"}
assert_false(self.exp._check_param_dict(param_dict))
param_dict = {"x": 1,
"name": "A"}
assert_true(self.exp._check_param_dict(param_dict))
class BasicExperimentAssistant(object):
"""
This ExperimentAssistant assists with executing experiments.
It provides methods for getting candidates to evaluate, returning the
evaluated Candidate and administrates the optimizer.
Attributes
----------
optimizer : Optimizer
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
optimizer_arguments : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
experiment : Experiment
The experiment this assistant assists with.
write_directory_base : string or None
The directory to write all results to. If not
given, a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int
States how often the csv file should be written to.
If set to 0 no results will be written.
logger : logging.logger
The logger for this class.
"""
AVAILABLE_STATUS = ["finished", "pausing", "working"]
optimizer = None
optimizer_arguments = None
experiment = None
write_directory_base = None
experiment_directory_base = None
csv_write_frequency = None
csv_steps_written = 0
logger = None
def __init__(self, name, optimizer, param_defs, experiment=None, optimizer_arguments=None,
minimization=True, write_directory_base="/tmp/APSIS_WRITING",
experiment_directory_base=None, csv_write_frequency=1):
"""
Initializes the BasicExperimentAssistant.
Parameters
----------
name : string
The name of the experiment. This does not have to be unique, but is
for human orientation.
optimizer : Optimizer instance or string
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
Alternatively, it can be a string corresponding to the optimizer,
as defined by apsis.utilities.optimizer_utils.
param_defs : dict of ParamDef.
This is the parameter space defining the experiment.
experiment : Experiment
Preinitialize this assistant with an existing experiment.
optimizer_arguments=None : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
minimization=True : bool
Whether the problem is one of minimization or maximization.
write_directory_base : string, optional
The global base directory for all writing. Will only be used
for creation of experiment_directory_base if this is not given.
experiment_directory_base : string or None, optional
The directory to write all the results to. If not
given a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int, optional
States how often the csv file should be written to.
If set to 0 no results will be written.
"""
self.logger = get_logger(self)
self.logger.info("Initializing experiment assistant.")
self.optimizer = optimizer
self.optimizer_arguments = optimizer_arguments
if experiment is None:
self.experiment = Experiment(name, param_defs, minimization)
else:
self.experiment = experiment
self.csv_write_frequency = csv_write_frequency
if self.csv_write_frequency != 0:
self.write_directory_base = write_directory_base
if experiment_directory_base is not None:
self.experiment_directory_base = experiment_directory_base
ensure_directory_exists(self.experiment_directory_base)
else:
self._create_experiment_directory()
self.logger.info("Experiment assistant successfully initialized.")
def get_next_candidate(self):
"""
Returns the Candidate next to evaluate.
Returns
-------
next_candidate : Candidate or None
The Candidate object that should be evaluated next. May be None.
"""
self.logger.info("Returning next candidate.")
self.optimizer = check_optimizer(self.optimizer,
optimizer_arguments=self.optimizer_arguments)
if not self.experiment.candidates_pending:
self.experiment.candidates_pending.extend(
self.optimizer.get_next_candidates(self.experiment))
next_candidate = self.experiment.candidates_pending.pop()
self.logger.info("next candidate found: %s" %next_candidate)
return next_candidate
def update(self, candidate, status="finished"):
"""
Updates the experiment_assistant with the status of an experiment
evaluation.
Parameters
----------
candidate : Candidate
The Candidate object whose status is updated.
status : {"finished", "pausing", "working"}
A string defining the status change. Can be one of the following:
- finished: The Candidate is now finished.
- pausing: The evaluation of Candidate has been paused and can be
resumed by another worker.
- working: The Candidate is now being worked on by a worker.
"""
if status not in self.AVAILABLE_STATUS:
message = ("status not in %s but %s."
%(str(self.AVAILABLE_STATUS), str(status)))
self.logger.error(message)
raise ValueError(message)
if not isinstance(candidate, Candidate):
message = ("candidate %s not a Candidate instance."
%str(candidate))
self.logger.error(message)
raise ValueError(message)
self.logger.info("Got new %s of candidate %s with parameters %s"
" and result %s" %(status, candidate, candidate.params,
candidate.result))
if status == "finished":
self.experiment.add_finished(candidate)
#Also delete all pending candidates from the experiment - we have
#new data available.
self.experiment.candidates_pending = []
#invoke the writing to files
step = len(self.experiment.candidates_finished)
if self.csv_write_frequency != 0 and step != 0 \
and step % self.csv_write_frequency == 0:
self._append_to_detailed_csv()
elif status == "pausing":
self.experiment.add_pausing(candidate)
elif status == "working":
self.experiment.add_working(candidate)
def get_best_candidate(self):
"""
Returns the best candidate to date.
Returns
-------
best_candidate : candidate or None
Returns a candidate if there is a best one (which corresponds to
at least one candidate evaluated) or None if none exists.
"""
return self.experiment.best_candidate
def _append_to_detailed_csv(self):
if len(self.experiment.candidates_finished) <= self.csv_steps_written:
return
#create file and header if
wHeader = False
if self.csv_steps_written == 0:
#set use header
wHeader = True
csv_string, steps_included = self.experiment.to_csv_results(wHeader=wHeader,
fromIndex=self.csv_steps_written)
#write
filename = os.path.join(self.experiment_directory_base,
self.experiment.name + "_results.csv")
with open(filename, 'a+') as detailed_file:
detailed_file.write(csv_string)
self.csv_steps_written += steps_included
def _create_experiment_directory(self):
global_start_date = time.time()
date_name = datetime.datetime.utcfromtimestamp(
global_start_date).strftime("%Y-%m-%d_%H:%M:%S")
self.experiment_directory_base = os.path.join(self.write_directory_base,
self.experiment.name + "_" + date_name)
ensure_directory_exists(self.experiment_directory_base)
class BasicExperimentAssistant(object):
"""
This ExperimentAssistant assists with executing experiments.
It provides methods for getting candidates to evaluate, returning the
evaluated Candidate and administrates the optimizer.
Attributes
----------
optimizer : Optimizer
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
optimizer_arguments : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
experiment : Experiment
The experiment this assistant assists with.
write_directory_base : string or None
The directory to write all results to. If not
given, a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int
States how often the csv file should be written to.
If set to 0 no results will be written.
logger : logging.logger
The logger for this class.
"""
AVAILABLE_STATUS = ["finished", "pausing", "working"]
optimizer = None
optimizer_arguments = None
experiment = None
write_directory_base = None
experiment_directory_base = None
csv_write_frequency = None
csv_steps_written = 0
logger = None
def __init__(
self,
name,
optimizer,
param_defs,
experiment=None,
optimizer_arguments=None,
minimization=True,
write_directory_base="/tmp/APSIS_WRITING",
experiment_directory_base=None,
csv_write_frequency=1,
):
"""
Initializes the BasicExperimentAssistant.
Parameters
----------
name : string
The name of the experiment. This does not have to be unique, but is
for human orientation.
optimizer : Optimizer instance or string
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
Alternatively, it can be a string corresponding to the optimizer,
as defined by apsis.utilities.optimizer_utils.
param_defs : dict of ParamDef.
This is the parameter space defining the experiment.
experiment : Experiment
Preinitialize this assistant with an existing experiment.
optimizer_arguments=None : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
minimization=True : bool
Whether the problem is one of minimization or maximization.
write_directory_base : string, optional
The global base directory for all writing. Will only be used
for creation of experiment_directory_base if this is not given.
experiment_directory_base : string or None, optional
The directory to write all the results to. If not
given a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int, optional
States how often the csv file should be written to.
If set to 0 no results will be written.
"""
self.logger = get_logger(self)
self.logger.info("Initializing experiment assistant.")
self.optimizer = optimizer
self.optimizer_arguments = optimizer_arguments
if experiment is None:
self.experiment = Experiment(name, param_defs, minimization)
else:
self.experiment = experiment
self.csv_write_frequency = csv_write_frequency
if self.csv_write_frequency != 0:
self.write_directory_base = write_directory_base
if experiment_directory_base is not None:
self.experiment_directory_base = experiment_directory_base
ensure_directory_exists(self.experiment_directory_base)
else:
self._create_experiment_directory()
self.logger.info("Experiment assistant successfully initialized.")
def get_next_candidate(self):
"""
Returns the Candidate next to evaluate.
Returns
-------
next_candidate : Candidate or None
The Candidate object that should be evaluated next. May be None.
"""
self.logger.info("Returning next candidate.")
self.optimizer = check_optimizer(self.optimizer, optimizer_arguments=self.optimizer_arguments)
if not self.experiment.candidates_pending:
self.experiment.candidates_pending.extend(self.optimizer.get_next_candidates(self.experiment))
next_candidate = self.experiment.candidates_pending.pop()
self.logger.info("next candidate found: %s" % next_candidate)
return next_candidate
def update(self, candidate, status="finished"):
"""
Updates the experiment_assistant with the status of an experiment
evaluation.
Parameters
----------
candidate : Candidate
The Candidate object whose status is updated.
status : {"finished", "pausing", "working"}
A string defining the status change. Can be one of the following:
- finished: The Candidate is now finished.
- pausing: The evaluation of Candidate has been paused and can be
resumed by another worker.
- working: The Candidate is now being worked on by a worker.
"""
if status not in self.AVAILABLE_STATUS:
message = "status not in %s but %s." % (str(self.AVAILABLE_STATUS), str(status))
self.logger.error(message)
raise ValueError(message)
if not isinstance(candidate, Candidate):
message = "candidate %s not a Candidate instance." % str(candidate)
self.logger.error(message)
raise ValueError(message)
self.logger.info(
"Got new %s of candidate %s with parameters %s"
" and result %s" % (status, candidate, candidate.params, candidate.result)
)
if status == "finished":
self.experiment.add_finished(candidate)
# Also delete all pending candidates from the experiment - we have
# new data available.
self.experiment.candidates_pending = []
# invoke the writing to files
step = len(self.experiment.candidates_finished)
if self.csv_write_frequency != 0 and step != 0 and step % self.csv_write_frequency == 0:
self._append_to_detailed_csv()
elif status == "pausing":
self.experiment.add_pausing(candidate)
elif status == "working":
self.experiment.add_working(candidate)
def get_best_candidate(self):
"""
Returns the best candidate to date.
Returns
-------
best_candidate : candidate or None
Returns a candidate if there is a best one (which corresponds to
at least one candidate evaluated) or None if none exists.
"""
return self.experiment.best_candidate
def _append_to_detailed_csv(self):
if len(self.experiment.candidates_finished) <= self.csv_steps_written:
return
# create file and header if
wHeader = False
if self.csv_steps_written == 0:
# set use header
wHeader = True
csv_string, steps_included = self.experiment.to_csv_results(wHeader=wHeader, fromIndex=self.csv_steps_written)
# write
filename = os.path.join(self.experiment_directory_base, self.experiment.name + "_results.csv")
with open(filename, "a+") as detailed_file:
detailed_file.write(csv_string)
self.csv_steps_written += steps_included
def _create_experiment_directory(self):
global_start_date = time.time()
date_name = datetime.datetime.utcfromtimestamp(global_start_date).strftime("%Y-%m-%d_%H:%M:%S")
self.experiment_directory_base = os.path.join(self.write_directory_base, self.experiment.name + "_" + date_name)
ensure_directory_exists(self.experiment_directory_base)
