def test_add_tracking_metrics(self):
experiment = make_experiment(
parameters=[{
"name": "x",
"type": "range",
"bounds": [0, 1]
}],
tracking_metric_names=None,
)
self.assertDictEqual(experiment._tracking_metrics, {})
metrics_names = ["metric_1", "metric_2"]
experiment = make_experiment(
parameters=[{
"name": "x",
"type": "range",
"bounds": [0, 1]
}],
tracking_metric_names=metrics_names,
)
self.assertDictEqual(
experiment._tracking_metrics,
{
metric_name: Metric(name=metric_name)
for metric_name in metrics_names
},
)
def test_objective_validation(self):
with self.assertRaisesRegex(UnsupportedError, "Ambiguous objective definition"):
make_experiment(
parameters={"name": "x", "type": "range", "bounds": [0, 1]},
objective_name="branin",
objectives={"branin": "minimize", "currin": "maximize"},
)
def with_evaluation_function(
parameters: List[TParameterRepresentation],
evaluation_function: TEvaluationFunction,
experiment_name: Optional[str] = None,
objective_name: Optional[str] = None,
minimize: bool = False,
parameter_constraints: Optional[List[str]] = None,
outcome_constraints: Optional[List[str]] = None,
total_trials: int = 20,
arms_per_trial: int = 1,
wait_time: int = 0,
random_seed: Optional[int] = None,
generation_strategy: Optional[GenerationStrategy] = None,
) -> "OptimizationLoop":
"""Constructs a synchronous `OptimizationLoop` using an evaluation
function."""
experiment = make_experiment(
name=experiment_name,
parameters=parameters,
objective_name=objective_name,
minimize=minimize,
parameter_constraints=parameter_constraints,
outcome_constraints=outcome_constraints,
)
return OptimizationLoop(
experiment=experiment,
total_trials=total_trials,
arms_per_trial=arms_per_trial,
random_seed=random_seed,
wait_time=wait_time,
generation_strategy=generation_strategy,
evaluation_function=evaluation_function,
)
def create_experiment(
self,
parameters: List[Dict[str, Union[TParamValue, List[TParamValue]]]],
name: Optional[str] = None,
objective_name: Optional[str] = None,
minimize: bool = False,
parameter_constraints: Optional[List[str]] = None,
outcome_constraints: Optional[List[str]] = None,
status_quo: Optional[TParameterization] = None,
) -> None:
"""Create a new experiment and save it if DBSettings available.
Args:
parameters: List of dictionaries representing parameters in the
experiment search space. Required elements in the dictionaries
are: "name" (name of this parameter, string), "type" (type of the
parameter: "range", "fixed", or "choice", string), and "bounds"
for range parameters (list of two values, lower bound first),
"values" for choice parameters (list of values), and "value" for
fixed parameters (single value).
objective: Name of the metric used as objective in this experiment.
This metric must be present in `raw_data` argument to `complete_trial`.
name: Name of the experiment to be created.
minimize: Whether this experiment represents a minimization problem.
parameter_constraints: List of string representation of parameter
constraints, such as "x3 >= x4" or "x3 + x4 + x5 >= 2". For sum
constraints, any number of arguments is accepted, and acceptable
operators are "<=" and ">=".
outcome_constraints: List of string representation of outcome
constraints of form "metric_name >= bound", like "m1 <= 3."
status_quo: Parameterization of the current state of the system.
If set, this will be added to each trial to be evaluated alongside
test configurations.
"""
if self.db_settings and not name:
raise ValueError( # pragma: no cover
"Must give the experiment a name if `db_settings` is not None."
)
self._experiment = make_experiment(
name=name,
parameters=parameters,
objective_name=objective_name,
minimize=minimize,
parameter_constraints=parameter_constraints,
outcome_constraints=outcome_constraints,
status_quo=status_quo,
)
if self._generation_strategy is None:
self._generation_strategy = choose_generation_strategy(
search_space=self._experiment.search_space,
enforce_sequential_optimization=self.
_enforce_sequential_optimization,
random_seed=self._random_seed,
)
self._save_experiment_and_generation_strategy_to_db_if_possible()
def create_experiment(
self,
parameters: List[Dict[str, Union[TParamValue, List[TParamValue]]]],
name: Optional[str] = None,
objective_name: Optional[str] = None,
minimize: bool = False,
parameter_constraints: Optional[List[str]] = None,
outcome_constraints: Optional[List[str]] = None,
) -> None:
"""Create a new experiment and save it if DBSettings available.
Args:
parameters: List of dictionaries representing parameters in the
experiment search space. Required elements in the dictionaries
are: "name" (name of this parameter, string), "type" (type of the
parameter: "range", "fixed", or "choice", string), and "bounds"
for range parameters (list of two values, lower bound first),
"values" for choice parameters (list of values), and "value" for
fixed parameters (single value).
objective: Name of the metric used as objective in this experiment.
This metric must be present in `raw_data` argument to `log_data`.
name: Name of the experiment to be created.
minimize: Whether this experiment represents a minimization problem.
parameter_constraints: List of string representation of parameter
constraints, such as "x3 >= x4" or "x3 + x4 >= 2".
outcome_constraints: List of string representation of outcome
constraints of form "metric_name >= bound", like "m1 <= 3."
"""
if self.db_settings and not name:
raise ValueError( # pragma: no cover
"Must give the experiment a name if `db_settings` is not None."
)
self._experiment = make_experiment(
name=name,
parameters=parameters,
objective_name=objective_name,
minimize=minimize,
parameter_constraints=parameter_constraints,
outcome_constraints=outcome_constraints,
)
if self.generation_strategy is None:
self.generation_strategy = choose_generation_strategy(
search_space=self._experiment.search_space,
enforce_sequential_optimization=self.
_enforce_sequential_optimization,
)
self._save_experiment_if_possible()
def setup_ax_experiment_optimizer(data, ax_search_domain, fixed_gbm_params,
control_arm_params):
ax_experiment = make_experiment(
name="GBM param optimisation",
parameters=ax_search_domain,
minimize=True,
status_quo=control_arm_params,
)
ax_trial_evaluation_func = partial(score_gbm_configuration, data,
fixed_gbm_params)
gbm_mae = construct_ax_metric(ax_trial_evaluation_func, 'gbm_mae')
ax_experiment.optimization_config = OptimizationConfig(objective=Objective(
metric=gbm_mae,
minimize=True,
), )
ax_experiment.runner = SimpleRunner()
return ax_experiment
def create_experiment(
self,
parameters: List[Dict[str, Union[TParamValue, List[TParamValue]]]],
name: Optional[str] = None,
objective_name: Optional[str] = None,
minimize: bool = False,
parameter_constraints: Optional[List[str]] = None,
outcome_constraints: Optional[List[str]] = None,
status_quo: Optional[TParameterization] = None,
overwrite_existing_experiment: bool = False,
experiment_type: Optional[str] = None,
choose_generation_strategy_kwargs: Optional[Dict[str, Any]] = None,
) -> None:
"""Create a new experiment and save it if DBSettings available.
Args:
parameters: List of dictionaries representing parameters in the
experiment search space. Required elements in the dictionaries
are: "name" (name of this parameter, string), "type" (type of the
parameter: "range", "fixed", or "choice", string), and "bounds"
for range parameters (list of two values, lower bound first),
"values" for choice parameters (list of values), and "value" for
fixed parameters (single value).
objective: Name of the metric used as objective in this experiment.
This metric must be present in `raw_data` argument to `complete_trial`.
name: Name of the experiment to be created.
minimize: Whether this experiment represents a minimization problem.
parameter_constraints: List of string representation of parameter
constraints, such as "x3 >= x4" or "-x3 + 2*x4 - 3.5*x5 >= 2". For
the latter constraints, any number of arguments is accepted, and
acceptable operators are "<=" and ">=".
outcome_constraints: List of string representation of outcome
constraints of form "metric_name >= bound", like "m1 <= 3."
status_quo: Parameterization of the current state of the system.
If set, this will be added to each trial to be evaluated alongside
test configurations.
overwrite_existing_experiment: If an experiment has already been set
on this `AxClient` instance, whether to reset it to the new one.
If overwriting the experiment, generation strategy will be
re-selected for the new experiment and restarted.
To protect experiments in production, one cannot overwrite existing
experiments if the experiment is already stored in the database,
regardless of the value of `overwrite_existing_experiment`.
choose_generation_strategy_kwargs: Keyword arguments to pass to
`choose_generation_strategy` function which determines what
generation strategy should be used when none was specified on init.
"""
if self.db_settings_set and not name:
raise ValueError( # pragma: no cover
"Must give the experiment a name if `db_settings` is not None."
)
if self.db_settings_set:
experiment_id, _ = self._get_experiment_and_generation_strategy_db_id(
experiment_name=not_none(name)
)
if experiment_id:
raise ValueError(
f"Experiment {name} already exists in the database. "
"To protect experiments that are running in production, "
"overwriting stored experiments is not allowed. To "
"start a new experiment and store it, change the "
"experiment's name."
)
if self._experiment is not None:
if overwrite_existing_experiment:
exp_name = self.experiment._name or "untitled"
new_exp_name = name or "untitled"
logger.info(
f"Overwriting existing experiment ({exp_name}) on this client "
f"with new experiment ({new_exp_name}) and restarting the "
"generation strategy."
)
self._generation_strategy = None
else:
raise ValueError(
"Experiment already created for this client instance. "
"Set the `overwrite_existing_experiment` to `True` to overwrite "
"with new experiment."
)
self._experiment = make_experiment(
name=name,
parameters=parameters,
objective_name=objective_name,
minimize=minimize,
parameter_constraints=parameter_constraints,
outcome_constraints=outcome_constraints,
status_quo=status_quo,
experiment_type=experiment_type,
)
try:
self._save_experiment_to_db_if_possible(
experiment=self.experiment,
suppress_all_errors=self._suppress_storage_errors,
)
except Exception:
# Unset the experiment on this `AxClient` instance if encountered and
# raising an error from saving the experiment, to avoid a case where
# overall `create_experiment` call fails with a storage error, but
# `self._experiment` is still set and user has to specify the
# `ooverwrite_existing_experiment` kwarg to re-attempt exp. creation.
self._experiment = None
raise
self._set_generation_strategy(
choose_generation_strategy_kwargs=choose_generation_strategy_kwargs
)
self._save_generation_strategy_to_db_if_possible(
generation_strategy=self.generation_strategy,
suppress_all_errors=self._suppress_storage_errors,
)
def create_experiment(
self,
parameters: List[Dict[str, Union[TParamValue, List[TParamValue]]]],
name: Optional[str] = None,
objective_name: Optional[str] = None,
minimize: bool = False,
parameter_constraints: Optional[List[str]] = None,
outcome_constraints: Optional[List[str]] = None,
status_quo: Optional[TParameterization] = None,
overwrite_existing_experiment: bool = False,
experiment_type: Optional[str] = None,
choose_generation_strategy_kwargs: Optional[Dict[str, Any]] = None,
) -> None:
"""Create a new experiment and save it if DBSettings available.
Args:
parameters: List of dictionaries representing parameters in the
experiment search space. Required elements in the dictionaries
are: "name" (name of this parameter, string), "type" (type of the
parameter: "range", "fixed", or "choice", string), and "bounds"
for range parameters (list of two values, lower bound first),
"values" for choice parameters (list of values), and "value" for
fixed parameters (single value).
objective: Name of the metric used as objective in this experiment.
This metric must be present in `raw_data` argument to `complete_trial`.
name: Name of the experiment to be created.
minimize: Whether this experiment represents a minimization problem.
parameter_constraints: List of string representation of parameter
constraints, such as "x3 >= x4" or "-x3 + 2*x4 - 3.5*x5 >= 2". For
the latter constraints, any number of arguments is accepted, and
acceptable operators are "<=" and ">=".
outcome_constraints: List of string representation of outcome
constraints of form "metric_name >= bound", like "m1 <= 3."
status_quo: Parameterization of the current state of the system.
If set, this will be added to each trial to be evaluated alongside
test configurations.
overwrite_existing_experiment: If an experiment has already been set
on this `AxClient` instance, whether to reset it to the new one.
If overwriting the experiment, generation strategy will be
re-selected for the new experiment and restarted.
choose_generation_strategy_kwargs: Keyword arguments to pass to
`choose_generation_strategy` function which determines what
generation strategy should be used when none was specified on init.
"""
if self.db_settings and not name:
raise ValueError( # pragma: no cover
"Must give the experiment a name if `db_settings` is not None."
)
if self.db_settings:
existing = None
try:
existing, _ = load_experiment_and_generation_strategy(
experiment_name=not_none(name),
db_settings=self.db_settings)
except ValueError: # Experiment does not exist, nothing to do.
pass
if existing and overwrite_existing_experiment:
logger.info(f"Overwriting existing experiment {name}.")
elif existing:
raise ValueError(
f"Experiment {name} exists; set the `overwrite_existing_"
"experiment` to `True` to overwrite with new experiment "
"or use `ax_client.load_experiment_from_database` to "
"continue an existing experiment.")
if self._experiment is not None:
if overwrite_existing_experiment:
exp_name = self.experiment._name or "untitled"
new_exp_name = name or "untitled"
logger.info(
f"Overwriting existing experiment ({exp_name}) on this client "
f"with new experiment ({new_exp_name}) and restarting the "
"generation strategy.")
self._generation_strategy = None
else:
raise ValueError(
f"Experiment already created for this client instance. "
"Set the `overwrite_existing_experiment` to `True` to overwrite "
"with new experiment.")
self._experiment = make_experiment(
name=name,
parameters=parameters,
objective_name=objective_name,
minimize=minimize,
parameter_constraints=parameter_constraints,
outcome_constraints=outcome_constraints,
status_quo=status_quo,
experiment_type=experiment_type,
)
choose_generation_strategy_kwargs = choose_generation_strategy_kwargs or {}
random_seed = choose_generation_strategy_kwargs.pop(
"random_seed", self._random_seed)
enforce_sequential_optimization = choose_generation_strategy_kwargs.pop(
"enforce_sequential_optimization",
self._enforce_sequential_optimization)
if self._generation_strategy is None:
self._generation_strategy = choose_generation_strategy(
search_space=not_none(self._experiment).search_space,
enforce_sequential_optimization=enforce_sequential_optimization,
random_seed=random_seed,
**choose_generation_strategy_kwargs,
)
self._save_experiment_and_generation_strategy_to_db_if_possible(
overwrite_existing_experiment=True)
def create_experiment(
self,
parameters: List[Dict[str, Union[TParamValue, List[TParamValue]]]],
name: Optional[str] = None,
objective_name: Optional[str] = None,
minimize: bool = False,
parameter_constraints: Optional[List[str]] = None,
outcome_constraints: Optional[List[str]] = None,
status_quo: Optional[TParameterization] = None,
overwrite_existing_experiment: bool = False,
experiment_type: Optional[str] = None,
) -> None:
"""Create a new experiment and save it if DBSettings available.
Args:
parameters: List of dictionaries representing parameters in the
experiment search space. Required elements in the dictionaries
are: "name" (name of this parameter, string), "type" (type of the
parameter: "range", "fixed", or "choice", string), and "bounds"
for range parameters (list of two values, lower bound first),
"values" for choice parameters (list of values), and "value" for
fixed parameters (single value).
objective: Name of the metric used as objective in this experiment.
This metric must be present in `raw_data` argument to `complete_trial`.
name: Name of the experiment to be created.
minimize: Whether this experiment represents a minimization problem.
parameter_constraints: List of string representation of parameter
constraints, such as "x3 >= x4" or "x3 + x4 + x5 >= 2". For sum
constraints, any number of arguments is accepted, and acceptable
operators are "<=" and ">=".
outcome_constraints: List of string representation of outcome
constraints of form "metric_name >= bound", like "m1 <= 3."
status_quo: Parameterization of the current state of the system.
If set, this will be added to each trial to be evaluated alongside
test configurations.
overwrite_existing_experiment: If `DBSettings` were provided on
instantiation and the experiment being created has the same name
as some experiment already stored, whether to overwrite the
existing experiment. Defaults to False.
"""
if self.db_settings and not name:
raise ValueError( # pragma: no cover
"Must give the experiment a name if `db_settings` is not None."
)
if self.db_settings:
existing = None
try:
existing, _ = load_experiment_and_generation_strategy(
experiment_name=not_none(name),
db_settings=self.db_settings)
except ValueError: # Experiment does not exist, nothing to do.
pass
if existing and overwrite_existing_experiment:
logger.info(f"Overwriting existing experiment {name}.")
elif existing:
raise ValueError(
f"Experiment {name} exists; set the `overwrite_existing_"
"experiment` to `True` to overwrite with new experiment "
"or use `ax_client.load_experiment_from_database` to "
"continue an existing experiment.")
self._experiment = make_experiment(
name=name,
parameters=parameters,
objective_name=objective_name,
minimize=minimize,
parameter_constraints=parameter_constraints,
outcome_constraints=outcome_constraints,
status_quo=status_quo,
experiment_type=experiment_type,
)
if self._generation_strategy is None:
self._generation_strategy = choose_generation_strategy(
search_space=self._experiment.search_space,
enforce_sequential_optimization=self.
_enforce_sequential_optimization,
random_seed=self._random_seed,
)
self._save_experiment_and_generation_strategy_to_db_if_possible(
overwrite_existing_experiment=True)
