def go(self):
"""Begin hyperparameter optimization process after experiment guidelines have been set and search dimensions are in place.
This process includes the following: setting the hyperparameter space; locating similar experiments to be used as
learning material for :class:`InformedOptimizationProtocol` s; and executing :meth:`_optimization_loop`, which actually
sets off the Experiment execution process"""
if self.model_initializer is None:
raise ValueError(
'Experiment guidelines and options must be set before hyperparameter optimization can be started'
)
self.logger = OptimizationReporter([_.name for _ in self.dimensions],
**self.reporter_parameters)
self.tested_keys = []
self._set_hyperparameter_space()
self._find_similar_experiments()
loop_start_time = datetime.now()
self._optimization_loop()
loop_end_time = datetime.now()
G.log_(
F'Optimization loop completed in {loop_end_time - loop_start_time}'
)
G.log_(
F'Best score was {self.best_score} from Experiment "{self.best_experiment}"'
)
def _validate_environment():
"""Check that there is a currently active and unoccupied Environment instance"""
if G.Env is None:
raise EnvironmentInactiveError()
if G.Env.current_task is None:
G.log_(f'Validated Environment with key: "{G.Env.cross_experiment_key}"')
else:
raise EnvironmentInvalidError("Must finish current task before starting a new one")
def _validate_environment():
"""Check that there is a currently active Environment instance that is not already occupied"""
if G.Env is None:
raise EnvironmentInactiveError()
if G.Env.current_task is None:
G.log_(
F'Validated Environment with key: "{G.Env.cross_experiment_key}"'
)
else:
raise EnvironmentInvalidError(
'A task is in progress. It must finish before a new one can be started'
)
def _optimization_loop(self, iteration=0):
"""Perform Experiment execution loop while `iteration` < `iterations`. At each iteration, an
Experiment will be executed, its results will be logged, and it will be compared to the
current best experiment
Parameters
----------
iteration: Int, default=0
The current iteration in the optimization loop"""
self.logger.print_optimization_header()
while iteration < self.iterations:
try:
self._execute_experiment()
except RepeatedExperimentError:
# G.debug_(F'Skipping repeated Experiment: {_ex!s}\n')
if len(self.similar_experiments) + len(
self.tested_keys) >= self.search_space_size:
G.log_(f"Hyperparameter search space has been exhausted")
break
self.skipped_iterations += 1
continue
except StopIteration:
if len(self.similar_experiments) + len(
self.tested_keys) >= self.search_space_size:
G.log_(f"Hyperparameter search space has been exhausted")
break
# G.debug_(f'Re-initializing hyperparameter grid after testing {len(self.tested_keys)} keys')
self._set_hyperparameter_space()
continue
self.logger.print_result(
self.current_hyperparameters_list,
self.current_score,
experiment_id=self.current_experiment.experiment_id,
)
if ((self.best_experiment is None) # First evaluation
or (self.do_maximize and
(self.best_score < self.current_score)) # New best max
or (not self.do_maximize and
(self.best_score > self.current_score)) # New best min
):
self.best_experiment = self.current_experiment.experiment_id
self.best_score = self.current_score
iteration += 1
def _optimization_loop(self, iteration=0):
"""Perform Experiment execution loop while `iteration` < `iterations`. At each iteration, an Experiment will be executed,
its results will be logged, and it will be compared to the current best experiment
Parameters
----------
iteration: Int, default=0
The current iteration in the optimization loop"""
self.logger.print_optimization_header()
while iteration < self.iterations:
try:
self._execute_experiment()
except RepeatedExperimentError:
# G.debug_(F'Skipping repeated Experiment: {_ex!s}\n')
self.skipped_iterations += 1
continue
except StopIteration:
if len(self.tested_keys) >= self.search_space_size:
G.log_(
F'Hyperparameter search space has been exhausted after testing {len(self.tested_keys)} keys'
)
break
# G.debug_(F'Re-initializing hyperparameter grid after testing {len(self.tested_keys)} keys')
self._set_hyperparameter_space()
continue
# TODO: :attr:`current_hyperparameters_list` only exists in Informed Protocols
self.logger.print_result(
self.current_hyperparameters_list,
self.current_score,
experiment_id=self.current_experiment.experiment_id)
if (self.best_experiment is None) or (self.current_score >
self.best_score):
self.best_experiment = self.current_experiment.experiment_id
self.best_score = self.current_score
iteration += 1
