def write_out_plots_current_step(self, same_steps_only=True):
"""
This method will write out all plots available to the path
configured in self.lab_run_directory.
Parameters
---------
same_steps_only : boolean, optional
Write only if all experiment assistants in this lab assistant
are currently in the same step.
"""
step_string, same_step = self._compute_current_step_overall()
if same_steps_only and not same_step:
return
plot_base = os.path.join(self.lab_run_directory, "plots")
plot_step_base = os.path.join(plot_base, step_string)
ensure_directory_exists(plot_step_base)
plots_to_write = self.generate_all_plots()
#finally write out all plots created above to their files
for plot_name in plots_to_write.keys():
plot_fig = plots_to_write[plot_name]
write_plot_to_file(plot_fig, plot_name + "_" + step_string, plot_step_base)
def _create_experiment_directory(self):
global_start_date = time.time()
date_name = datetime.datetime.utcfromtimestamp(global_start_date).strftime("%Y-%m-%d_%H:%M:%S")
self.experiment_directory_base = os.path.join(self.write_directory_base, self.experiment.name + "_" + date_name)
ensure_directory_exists(self.experiment_directory_base)
def get_logger(module, specific_log_name=None):
"""
Abstraction from logging.getLogging, which also adds initialization.
Logging is configured directly at root level (in the standard usecase, at
least). You also have the opportunity to specify a certain directory to
which details of only this logger (and all subloggers) are written.
Currently, nothing is configurable from the outside. This is planned to be
changed.
Parameters
----------
module : object or string
The object for which we'd like to get the logger. The name of the
logger is then, analogous to logging, set to
module.__module__ + "." + module.__class__.__name__
If the object is a string it will be taken as name directly.
specific_log_name : string, optional
If you want logging for this logger (and all sublogger) to a specific
file, this allows you to set the corresponding filename.
Returns
-------
logger: logging.logger
A logging for module.
"""
#if logger is already given as a string take directly. otherwise compute.
if isinstance(module, basestring):
new_logger_name = module
else:
new_logger_name = module.__module__ + "." + module.__class__.__name__
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
LOG_ROOT = os.environ.get('APSIS_LOG_ROOT', '/tmp/APSIS_WRITING/logs')
ensure_directory_exists(LOG_ROOT)
global logging_intitialized
if not logging_intitialized:
logging_intitialized = True
#initialize the root logger.
project_dirname = os.path.dirname(apsis.__file__)
log_config_file = os.path.join(project_dirname, 'config/logging.conf')
logging.config.fileConfig(
log_config_file,
defaults={'logfilename': os.path.join(LOG_ROOT, "log")})
logger_existed = False
if new_logger_name in logging.Logger.manager.loggerDict:
logger_existed = True
logger = logging.getLogger(new_logger_name)
if specific_log_name is not None and not logger_existed:
fh = logging.FileHandler(os.path.join(LOG_ROOT, specific_log_name))
fh.setFormatter(formatter)
logger.addHandler(fh)
return logger
def __init__(self, name, optimizer, param_defs, experiment=None, optimizer_arguments=None,
minimization=True, write_directory_base="/tmp/APSIS_WRITING",
experiment_directory_base=None, csv_write_frequency=1):
"""
Initializes the BasicExperimentAssistant.
Parameters
----------
name : string
The name of the experiment. This does not have to be unique, but is
for human orientation.
optimizer : Optimizer instance or string
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
Alternatively, it can be a string corresponding to the optimizer,
as defined by apsis.utilities.optimizer_utils.
param_defs : dict of ParamDef.
This is the parameter space defining the experiment.
experiment : Experiment
Preinitialize this assistant with an existing experiment.
optimizer_arguments=None : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
minimization=True : bool
Whether the problem is one of minimization or maximization.
write_directory_base : string, optional
The global base directory for all writing. Will only be used
for creation of experiment_directory_base if this is not given.
experiment_directory_base : string or None, optional
The directory to write all the results to. If not
given a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int, optional
States how often the csv file should be written to.
If set to 0 no results will be written.
"""
self.logger = get_logger(self)
self.logger.info("Initializing experiment assistant.")
self.optimizer = optimizer
self.optimizer_arguments = optimizer_arguments
if experiment is None:
self.experiment = Experiment(name, param_defs, minimization)
else:
self.experiment = experiment
self.csv_write_frequency = csv_write_frequency
if self.csv_write_frequency != 0:
self.write_directory_base = write_directory_base
if experiment_directory_base is not None:
self.experiment_directory_base = experiment_directory_base
ensure_directory_exists(self.experiment_directory_base)
else:
self._create_experiment_directory()
self.logger.info("Experiment assistant successfully initialized.")
def _create_experiment_directory(self):
global_start_date = time.time()
date_name = datetime.datetime.utcfromtimestamp(
global_start_date).strftime("%Y-%m-%d_%H:%M:%S")
self.experiment_directory_base = os.path.join(self.write_directory_base,
self.experiment.name + "_" + date_name)
ensure_directory_exists(self.experiment_directory_base)
def get_logger(module, specific_log_name=None):
"""
Abstraction from logging.getLogging, which also adds initialization.
Logging is configured directly at root level (in the standard usecase, at
least). You also have the opportunity to specify a certain directory to
which details of only this logger (and all subloggers) are written.
Currently, nothing is configurable from the outside. This is planned to be
changed.
Parameters
----------
module : object or string
The object for which we'd like to get the logger. The name of the
logger is then, analogous to logging, set to
module.__module__ + "." + module.__class__.__name__
If the object is a string it will be taken as name directly.
specific_log_name : string, optional
If you want logging for this logger (and all sublogger) to a specific
file, this allows you to set the corresponding filename.
Returns
-------
logger: logging.logger
A logging for module.
"""
#if logger is already given as a string take directly. otherwise compute.
if isinstance(module, basestring):
new_logger_name = module
else:
new_logger_name = module.__module__ + "." + module.__class__.__name__
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
LOG_ROOT = os.environ.get('APSIS_LOG_ROOT', '/tmp/APSIS_WRITING/logs')
ensure_directory_exists(LOG_ROOT)
global logging_intitialized
if not logging_intitialized:
logging_intitialized = True
#initialize the root logger.
project_dirname = os.path.dirname(apsis.__file__)
log_config_file = os.path.join(project_dirname, 'config/logging.conf')
logging.config.fileConfig(log_config_file, defaults={'logfilename': os.path.join(LOG_ROOT, "log")})
logger_existed = False
if new_logger_name in logging.Logger.manager.loggerDict:
logger_existed = True
logger = logging.getLogger(new_logger_name)
if specific_log_name is not None and not logger_existed:
fh = logging.FileHandler(os.path.join(LOG_ROOT, specific_log_name))
fh.setFormatter(formatter)
logger.addHandler(fh)
return logger
def _create_experiment_directory(self):
"""
Generates an experiment directory from the base write directory.
"""
global_start_date = time.time()
date_name = datetime.datetime.utcfromtimestamp(
global_start_date).strftime("%Y-%m-%d_%H:%M:%S")
self._experiment_directory_base = os.path.join(
self._write_directory_base,
self._experiment.exp_id)
ensure_directory_exists(self._experiment_directory_base)
def _init_directory_structure(self):
"""
Method to create the directory structure if not exists
for results and plots writing
"""
if self.lab_run_directory is None:
date_name = datetime.datetime.utcfromtimestamp(
self.global_start_date).strftime("%Y-%m-%d_%H:%M:%S")
self.lab_run_directory = os.path.join(self.write_directory_base,
date_name)
ensure_directory_exists(self.lab_run_directory)
def write_plots(self):
"""
Writes out the plots of this assistant.
"""
fig = self.plot_result_per_step()
filename = "result_per_step_%i" \
% len(self._experiment.candidates_finished)
path = self._experiment_directory_base + "/plots"
ensure_directory_exists(path)
write_plot_to_file(fig, filename, path)
write_plot_to_file(fig, "cur_state", self._experiment_directory_base)
plt.close(fig)
def __init__(self, optimizer_class, optimizer_arguments=None,
write_directory_base=None, experiment_directory=None,
csv_write_frequency=1):
"""
Initializes this experiment assistant.
Note that calling this function does not yet create an experiment, for
that, use init_experiment. If there is an already existing experiment,
you can just set self._experiment.
Parameters
----------
optimizer_class : subclass of Optimizer
The class of the optimizer, used to initialize it.
optimizer_arguments : dict, optional
The dictionary of optimizer arguments. If None, default values will
be used.
experiment_directory_base : string, optional
The folder to which the csv intermediary results and the plots will
be written. Default is <write_directory_base>/exp_id.
write_directory_base : string, optional
The base directory. In the default case, this is dependant on the
OS. On windows, it is set to ./APSIS_WRITING/. On Linux,
it is set to /tmp/APSIS_WRITING/. If an
experiment_directory has been given, this will be ignored.
csv_write_frequency : int, optional
This sets the frequency with which the csv file is written. If set
to 1 (the default), it writes every step. If set to 2, every second
and so on. Note that it still writes out every step eventually.
"""
self._logger = get_logger(self)
self._logger.info("Initializing experiment assistant.")
self._csv_write_frequency = csv_write_frequency
self._optimizer = optimizer_class
self._optimizer_arguments = optimizer_arguments
if self._csv_write_frequency != 0:
if experiment_directory is not None:
self._experiment_directory_base = experiment_directory
ensure_directory_exists(self._experiment_directory_base)
else:
if write_directory_base is None:
if os.name == "nt":
self._write_directory_base = \
os.path.relpath("APSIS_WRITING")
else:
self._write_directory_base = "/tmp/APSIS_WRITING"
else:
self._write_directory_base = write_directory_base
self._logger.info("Experiment assistant for successfully "
"initialized.")
def _load_exp_assistant_from_path(self, path):
"""
This loads a complete exp_assistant from path.
Specifically, it looks for exp_assistant.json in the path and restores
optimizer_class, optimizer_arguments and write_dir from this. It then
loads the experiment from the write_dir/experiment.json, then
initializes both.
Parameters
----------
path : string
The path from which to initialize. This must contain an
exp_assistant.json as specified.
"""
self._logger.debug("Loading Exp_assistant from path %s" % path)
with open(path + "/exp_assistant.json", 'r') as infile:
exp_assistant_json = json.load(infile)
optimizer_class = exp_assistant_json["optimizer_class"]
optimizer_arguments = exp_assistant_json["optimizer_arguments"]
exp_ass_write_dir = exp_assistant_json["write_dir"]
ensure_directory_exists(exp_ass_write_dir)
self._logger.debug(
"\tLoaded exp_parameters: "
"optimizer_class: %s, optimizer_arguments: %s,"
"write_dir: %s" %
(optimizer_class, optimizer_arguments, exp_ass_write_dir))
exp = self._load_experiment(path)
self._logger.debug("\tLoaded Experiment. %s" % exp.to_dict())
exp_ass = ExperimentAssistant(optimizer_class=optimizer_class,
experiment=exp,
optimizer_arguments=optimizer_arguments,
write_dir=exp_ass_write_dir)
if exp_ass.exp_id in self._exp_assistants:
raise ValueError("Loaded exp_id is duplicated in experiment! id "
"is %s" % exp_ass.exp_id)
self._exp_assistants[exp_ass.exp_id] = exp_ass
self._logger.info("Successfully loaded experiment from %s." % path)
def write_out_plots_current_step(self, exp_ass=None, same_steps_only=True):
"""
This method will write out all plots available to the path
configured in self.lab_run_directory.
Parameters
---------
exp_ass : list, optional
List of experiment assistant names to include in the plots. Defaults to
None, which is equivalent to all.
same_steps_only : boolean, optional
Write only if all experiment assistants in this lab assistant
are currently in the same step.
"""
min_step = self._get_min_step()
if same_steps_only:
plot_up_to = min_step
else:
plot_up_to = None
plot_base = os.path.join(self._lab_run_directory, "plots")
plot_step_base = os.path.join(plot_base, "step_" + str(min_step))
ensure_directory_exists(plot_step_base)
if exp_ass is None:
exp_ass = self._exp_assistants.keys()
plots_to_write = self.generate_all_plots(exp_ass, plot_up_to)
#finally write out all plots created above to their files
for plot_name in plots_to_write.keys():
plot_fig = plots_to_write[plot_name]
write_plot_to_file(plot_fig, plot_name + "_step" + str(min_step),
plot_step_base)
plt.close(plot_fig)
def init_experiment(self,
name,
optimizer,
param_defs,
exp_id=None,
notes=None,
optimizer_arguments=None,
minimization=True):
"""
Initializes an experiment.
Parameters
----------
name : string
name of the experiment.
optimizer : string
String representation of the optimizer.
param_defs : dict of parameter definitions
Dictionary of parameter definition classes.
optimizer_arguments : dict, optional
A dictionary defining the operation of the optimizer. See the
respective documentation of the optimizers.
Default is None, which are default values.
exp_id : string or None, optional
The id of the experiment, which will be used to reference it.
Should be a proper uuid, and especially has to be unique. If it is
not, an error may be returned.
notes : jsonable object or None, optional
Any note that you'd like to put in the experiment. Could be used
to provide some details on the experiment, on the start time or the
user starting it.
minimization : bool, optional
Whether the problem is one of minimization. Defaults to True.
Returns
-------
exp_id : string
String representing the id of the experiment or "failed" if failed.
Raises
------
ValueError :
Iff there already is an experiment with the exp_id for this lab
assistant. Does not occur if no exp_id is given.
"""
self._logger.debug("Initializing new experiment. Parameters: "
"name: %s, optimizer: %s, param_defs: %s, "
"exp_id: %s, notes: %s, optimizer_arguments: %s, "
"minimization: %s" %
(name, optimizer, param_defs, exp_id, notes,
optimizer_arguments, minimization))
if exp_id in self._exp_assistants.keys():
raise ValueError("Already an experiment with id %s registered." %
exp_id)
if exp_id is None:
while True:
exp_id = uuid.uuid4().hex
if exp_id not in self._exp_assistants.keys():
break
self._logger.debug("\tGenerated new exp_id: %s" % exp_id)
if not self._write_dir:
exp_assistant_write_directory = None
else:
exp_assistant_write_directory = os.path.join(self._write_dir +
"/" + exp_id)
ensure_directory_exists(exp_assistant_write_directory)
self._logger.debug("\tExp_ass directory: %s" %
exp_assistant_write_directory)
exp = experiment.Experiment(name, param_defs, exp_id, notes,
minimization)
exp_ass = ExperimentAssistant(optimizer,
experiment=exp,
optimizer_arguments=optimizer_arguments,
write_dir=exp_assistant_write_directory)
self._exp_assistants[exp_id] = exp_ass
self._logger.info("Experiment initialized successfully with id %s." %
exp_id)
self._write_state_to_file()
return exp_id
def get_logger(module, extra_info=None, save_path=None):
"""
Abstraction from logging.getLogging, which also adds initialization.
This loads the logging config from config/logging.conf.
Parameters
----------
module : object or string
The object for which we'd like to get the logger. The name of the
logger is then, analogous to logging, set to
module.__module__ + "." + module.__class__.__name__
If the object is a string it will be taken as name directly.
extra_info : string, optional
If None (the default), a usual logger is returned. If not, a
logger_adapter is returned, which always prepends the corresponding
string.
save_path : string, optional
The path on which to store the logging. If logging has been initialized
previously, this is ignored (and a warning is logged). If a path has
been specified in the config file, this is also ignored (and a warning
is issued). Otherwise, this path replaces all instances of the token
<SAVE_PATH> in the file_name of all handlers.
If it does not end with "/" we'll automatically add it. That means both
"/tmp/APSIS_WRITING" and "/tmp/APSIS_WRITING/" is treated identically,
and logging is added in "/tmp/APSIS_WRITING/logs".
Returns
-------
logger: logging.logger
A logging for module.
"""
#if logger is already given as a string take directly. otherwise compute.
if isinstance(module, basestring):
new_logger_name = module
else:
new_logger_name = module.__module__ + "." + module.__class__.__name__
global testing
global logging_intitialized
if not logging_intitialized and not testing:
logging_intitialized = True
# Look for the logging config file.
project_dirname = os.path.dirname(apsis.__file__)
log_config_file = os.path.join(project_dirname, 'config/logging.conf')
with open(log_config_file, "r") as conf_file:
conf_dict = yaml.load(conf_file)
handlers = conf_dict["handlers"]
handler_keys = handlers.keys()
for h in handler_keys:
if "filename" in handlers[h]:
if "<SAVE_PATH>" in handlers[h]["filename"]:
if not save_path.endswith("/"):
save_path += "/"
handlers[h]["filename"] = handlers[h]["filename"].replace(
"<SAVE_PATH>/", save_path).replace("<SAVE_PATH>",
save_path)
ensure_directory_exists(os.path.dirname(handlers[h]["filename"]))
logging.config.dictConfig(conf_dict)
logger = logging.getLogger(new_logger_name)
if extra_info:
logger = AddInfoClass(logger, {"extra_info": extra_info})
return logger
def __init__(
self,
name,
optimizer,
param_defs,
experiment=None,
optimizer_arguments=None,
minimization=True,
write_directory_base="/tmp/APSIS_WRITING",
experiment_directory_base=None,
csv_write_frequency=1,
):
"""
Initializes the BasicExperimentAssistant.
Parameters
----------
name : string
The name of the experiment. This does not have to be unique, but is
for human orientation.
optimizer : Optimizer instance or string
This is an optimizer implementing the corresponding functions: It
gets an experiment instance, and returns one or multiple candidates
which should be evaluated next.
Alternatively, it can be a string corresponding to the optimizer,
as defined by apsis.utilities.optimizer_utils.
param_defs : dict of ParamDef.
This is the parameter space defining the experiment.
experiment : Experiment
Preinitialize this assistant with an existing experiment.
optimizer_arguments=None : dict
These are arguments for the optimizer. Refer to their documentation
as to which are available.
minimization=True : bool
Whether the problem is one of minimization or maximization.
write_directory_base : string, optional
The global base directory for all writing. Will only be used
for creation of experiment_directory_base if this is not given.
experiment_directory_base : string or None, optional
The directory to write all the results to. If not
given a directory with timestamp will automatically be created
in write_directory_base
csv_write_frequency : int, optional
States how often the csv file should be written to.
If set to 0 no results will be written.
"""
self.logger = get_logger(self)
self.logger.info("Initializing experiment assistant.")
self.optimizer = optimizer
self.optimizer_arguments = optimizer_arguments
if experiment is None:
self.experiment = Experiment(name, param_defs, minimization)
else:
self.experiment = experiment
self.csv_write_frequency = csv_write_frequency
if self.csv_write_frequency != 0:
self.write_directory_base = write_directory_base
if experiment_directory_base is not None:
self.experiment_directory_base = experiment_directory_base
ensure_directory_exists(self.experiment_directory_base)
else:
self._create_experiment_directory()
self.logger.info("Experiment assistant successfully initialized.")
