def test_save_load_experiment_and_generation_strategy(self):
exp, gs = load_experiment_and_generation_strategy(
self.exp.name, self.db_settings)
self.assertIsNone(gs)
gs = get_generation_strategy()
gs._experiment = self.exp
save_experiment_and_generation_strategy(self.exp, gs, self.db_settings)
exp, gs = load_experiment_and_generation_strategy(
self.exp.name, self.db_settings)
self.assertIsNotNone(gs)
def load_experiment_from_database(
self,
experiment_name: str,
choose_generation_strategy_kwargs: Optional[Dict[str, Any]] = None,
) -> None:
"""Load an existing experiment from database using the `DBSettings`
passed to this `AxClient` on instantiation.
Args:
experiment_name: Name of the experiment.
Returns:
Experiment object.
"""
if not self.db_settings:
raise ValueError( # pragma: no cover
"Cannot load an experiment in the absence of the DB settings."
"Please initialize `AxClient` with DBSettings."
)
experiment, generation_strategy = load_experiment_and_generation_strategy(
experiment_name=experiment_name, db_settings=self.db_settings
)
self._experiment = experiment
logger.info(f"Loaded {experiment}.")
if generation_strategy is None: # pragma: no cover
self._set_generation_strategy(
choose_generation_strategy_kwargs=choose_generation_strategy_kwargs
)
else:
self._generation_strategy = generation_strategy
logger.info(
f"Using generation strategy associated with the loaded experiment:"
f" {generation_strategy}."
)
def load_experiment_from_database(self, experiment_name: str) -> None:
"""Load an existing experiment from database using the `DBSettings`
passed to this `AxClient` on instantiation.
Args:
experiment_name: Name of the experiment.
Returns:
Experiment object.
"""
if not self.db_settings:
raise ValueError( # pragma: no cover
"Cannot load an experiment in the absence of the DB settings."
"Please initialize `AxClient` with DBSettings.")
experiment, generation_strategy = load_experiment_and_generation_strategy(
experiment_name=experiment_name, db_settings=self.db_settings)
self._experiment = experiment
logger.info(f"Loaded {experiment}.")
if generation_strategy is None: # pragma: no cover
self._generation_strategy = choose_generation_strategy(
# pyre-fixme[16]: `Optional` has no attribute `search_space`.
search_space=self._experiment.search_space,
enforce_sequential_optimization=self.
_enforce_sequential_optimization,
random_seed=self._random_seed,
)
else:
self._generation_strategy = generation_strategy
logger.info(
f"Using generation strategy associated with the loaded experiment:"
f" {generation_strategy}.")
def load_experiment_from_database(self, experiment_name: str) -> None:
"""Load an existing experiment from database using the `DBSettings`
passed to this `AxClient` on instantiation.
Args:
experiment_name: Name of the experiment.
Returns:
Experiment object.
"""
if not self.db_settings:
raise ValueError( # pragma: no cover
"Cannot load an experiment in the absence of the DB settings."
"Please initialize `AxClient` with DBSettings.")
experiment, generation_strategy = load_experiment_and_generation_strategy(
experiment_name=experiment_name, db_settings=self.db_settings)
self._experiment = experiment
if generation_strategy is None: # pragma: no cover
self._generation_strategy = choose_generation_strategy(
search_space=self._experiment.search_space,
enforce_sequential_optimization=self.
_enforce_sequential_optimization,
random_seed=self._random_seed,
)
else:
self._generation_strategy = generation_strategy
def _load_experiment_and_generation_strategy(
self, experiment_name: str
) -> Tuple[Optional[Experiment], Optional[GenerationStrategy]]:
"""Loads experiment and its corresponding generation strategy from database
if DB settings are set on this `WithDBSettingsBase` instance.
Args:
experiment_name: Name of the experiment to load, used as unique
identifier by which to find the experiment.
Returns:
- Tuple of `None` and `None` if `DBSettings` are set and no experiment
exists by the given name.
- Tuple of `Experiment` and `None` if experiment exists but does not
have a generation strategy attached to it.
- Tuple of `Experiment` and `GenerationStrategy` if experiment exists
and has a generation strategy attached to it.
"""
if not self.db_settings_set:
raise ValueError("Cannot load from DB in absence of DB settings.")
try:
return load_experiment_and_generation_strategy(
experiment_name=experiment_name, db_settings=self.db_settings
)
except ValueError:
return None, None
def load_experiment(self, experiment_name: str) -> None:
"""[Work in progress] Load an existing experiment.
Args:
experiment_name: Name of the experiment.
Returns:
Experiment object.
"""
if not self.db_settings:
raise ValueError( # pragma: no cover
"Cannot load an experiment in the absence of the DB settings."
"Please initialize `AxClient` with DBSettings.")
experiment, generation_strategy = load_experiment_and_generation_strategy(
experiment_name=experiment_name, db_settings=self.db_settings)
self._experiment = experiment
self.generation_strategy = generation_strategy
def create_experiment(
self,
parameters: List[Dict[str, Union[TParamValue, List[TParamValue]]]],
name: Optional[str] = None,
objective_name: Optional[str] = None,
minimize: bool = False,
parameter_constraints: Optional[List[str]] = None,
outcome_constraints: Optional[List[str]] = None,
status_quo: Optional[TParameterization] = None,
overwrite_existing_experiment: bool = False,
experiment_type: Optional[str] = None,
choose_generation_strategy_kwargs: Optional[Dict[str, Any]] = None,
) -> None:
"""Create a new experiment and save it if DBSettings available.
Args:
parameters: List of dictionaries representing parameters in the
experiment search space. Required elements in the dictionaries
are: "name" (name of this parameter, string), "type" (type of the
parameter: "range", "fixed", or "choice", string), and "bounds"
for range parameters (list of two values, lower bound first),
"values" for choice parameters (list of values), and "value" for
fixed parameters (single value).
objective: Name of the metric used as objective in this experiment.
This metric must be present in `raw_data` argument to `complete_trial`.
name: Name of the experiment to be created.
minimize: Whether this experiment represents a minimization problem.
parameter_constraints: List of string representation of parameter
constraints, such as "x3 >= x4" or "-x3 + 2*x4 - 3.5*x5 >= 2". For
the latter constraints, any number of arguments is accepted, and
acceptable operators are "<=" and ">=".
outcome_constraints: List of string representation of outcome
constraints of form "metric_name >= bound", like "m1 <= 3."
status_quo: Parameterization of the current state of the system.
If set, this will be added to each trial to be evaluated alongside
test configurations.
overwrite_existing_experiment: If an experiment has already been set
on this `AxClient` instance, whether to reset it to the new one.
If overwriting the experiment, generation strategy will be
re-selected for the new experiment and restarted.
To protect experiments in production, one cannot overwrite existing
experiments if the experiment is already stored in the database,
regardless of the value of `overwrite_existing_experiment`.
choose_generation_strategy_kwargs: Keyword arguments to pass to
`choose_generation_strategy` function which determines what
generation strategy should be used when none was specified on init.
"""
if self.db_settings and not name:
raise ValueError( # pragma: no cover
"Must give the experiment a name if `db_settings` is not None."
)
if self.db_settings:
existing = None
try:
existing, _ = load_experiment_and_generation_strategy(
experiment_name=not_none(name), db_settings=self.db_settings
)
except ValueError: # Experiment does not exist, nothing to do.
pass
if existing:
raise ValueError(
f"Experiment {name} already exists in the database. "
"To protect experiments that are running in production, "
"overwriting stored experiments is not allowed. To "
"start a new experiment and store it, change the "
"experiment's name."
)
if self._experiment is not None:
if overwrite_existing_experiment:
exp_name = self.experiment._name or "untitled"
new_exp_name = name or "untitled"
logger.info(
f"Overwriting existing experiment ({exp_name}) on this client "
f"with new experiment ({new_exp_name}) and restarting the "
"generation strategy."
)
self._generation_strategy = None
else:
raise ValueError(
f"Experiment already created for this client instance. "
"Set the `overwrite_existing_experiment` to `True` to overwrite "
"with new experiment."
)
self._experiment = make_experiment(
name=name,
parameters=parameters,
objective_name=objective_name,
minimize=minimize,
parameter_constraints=parameter_constraints,
outcome_constraints=outcome_constraints,
status_quo=status_quo,
experiment_type=experiment_type,
)
self._set_generation_strategy(
choose_generation_strategy_kwargs=choose_generation_strategy_kwargs
)
self._save_experiment_to_db_if_possible(
suppress_all_errors=self._suppress_storage_errors
)
self._save_generation_strategy_to_db_if_possible(
suppress_all_errors=self._suppress_storage_errors
)
def create_experiment(
self,
parameters: List[Dict[str, Union[TParamValue, List[TParamValue]]]],
name: Optional[str] = None,
objective_name: Optional[str] = None,
minimize: bool = False,
parameter_constraints: Optional[List[str]] = None,
outcome_constraints: Optional[List[str]] = None,
status_quo: Optional[TParameterization] = None,
overwrite_existing_experiment: bool = False,
experiment_type: Optional[str] = None,
choose_generation_strategy_kwargs: Optional[Dict[str, Any]] = None,
) -> None:
"""Create a new experiment and save it if DBSettings available.
Args:
parameters: List of dictionaries representing parameters in the
experiment search space. Required elements in the dictionaries
are: "name" (name of this parameter, string), "type" (type of the
parameter: "range", "fixed", or "choice", string), and "bounds"
for range parameters (list of two values, lower bound first),
"values" for choice parameters (list of values), and "value" for
fixed parameters (single value).
objective: Name of the metric used as objective in this experiment.
This metric must be present in `raw_data` argument to `complete_trial`.
name: Name of the experiment to be created.
minimize: Whether this experiment represents a minimization problem.
parameter_constraints: List of string representation of parameter
constraints, such as "x3 >= x4" or "-x3 + 2*x4 - 3.5*x5 >= 2". For
the latter constraints, any number of arguments is accepted, and
acceptable operators are "<=" and ">=".
outcome_constraints: List of string representation of outcome
constraints of form "metric_name >= bound", like "m1 <= 3."
status_quo: Parameterization of the current state of the system.
If set, this will be added to each trial to be evaluated alongside
test configurations.
overwrite_existing_experiment: If `DBSettings` were provided on
instantiation and the experiment being created has the same name
as some experiment already stored, whether to overwrite the
existing experiment. Defaults to False.
choose_generation_strategy_kwargs: Keyword arguments to pass to
`choose_generation_strategy` function which determines what
generation strategy should be used when none was specified on init.
"""
if self.db_settings and not name:
raise ValueError( # pragma: no cover
"Must give the experiment a name if `db_settings` is not None."
)
if self.db_settings:
existing = None
try:
existing, _ = load_experiment_and_generation_strategy(
experiment_name=not_none(name),
db_settings=self.db_settings)
except ValueError: # Experiment does not exist, nothing to do.
pass
if existing and overwrite_existing_experiment:
logger.info(f"Overwriting existing experiment {name}.")
elif existing:
raise ValueError(
f"Experiment {name} exists; set the `overwrite_existing_"
"experiment` to `True` to overwrite with new experiment "
"or use `ax_client.load_experiment_from_database` to "
"continue an existing experiment.")
self._experiment = make_experiment(
name=name,
parameters=parameters,
objective_name=objective_name,
minimize=minimize,
parameter_constraints=parameter_constraints,
outcome_constraints=outcome_constraints,
status_quo=status_quo,
experiment_type=experiment_type,
)
if self._generation_strategy is None:
self._generation_strategy = choose_generation_strategy(
# pyre-fixme[16]: `Optional` has no attribute `search_space`.
search_space=self._experiment.search_space,
enforce_sequential_optimization=self.
_enforce_sequential_optimization,
random_seed=self._random_seed,
)
self._save_experiment_and_generation_strategy_to_db_if_possible(
overwrite_existing_experiment=True)
