def test_save_load_experiment_and_generation_strategy(self): exp, gs = load_experiment_and_generation_strategy( self.exp.name, self.db_settings) self.assertIsNone(gs) gs = get_generation_strategy() gs._experiment = self.exp save_experiment_and_generation_strategy(self.exp, gs, self.db_settings) exp, gs = load_experiment_and_generation_strategy( self.exp.name, self.db_settings) self.assertIsNotNone(gs)
def load_experiment_from_database( self, experiment_name: str, choose_generation_strategy_kwargs: Optional[Dict[str, Any]] = None, ) -> None: """Load an existing experiment from database using the `DBSettings` passed to this `AxClient` on instantiation. Args: experiment_name: Name of the experiment. Returns: Experiment object. """ if not self.db_settings: raise ValueError( # pragma: no cover "Cannot load an experiment in the absence of the DB settings." "Please initialize `AxClient` with DBSettings." ) experiment, generation_strategy = load_experiment_and_generation_strategy( experiment_name=experiment_name, db_settings=self.db_settings ) self._experiment = experiment logger.info(f"Loaded {experiment}.") if generation_strategy is None: # pragma: no cover self._set_generation_strategy( choose_generation_strategy_kwargs=choose_generation_strategy_kwargs ) else: self._generation_strategy = generation_strategy logger.info( f"Using generation strategy associated with the loaded experiment:" f" {generation_strategy}." )
def load_experiment_from_database(self, experiment_name: str) -> None: """Load an existing experiment from database using the `DBSettings` passed to this `AxClient` on instantiation. Args: experiment_name: Name of the experiment. Returns: Experiment object. """ if not self.db_settings: raise ValueError( # pragma: no cover "Cannot load an experiment in the absence of the DB settings." "Please initialize `AxClient` with DBSettings.") experiment, generation_strategy = load_experiment_and_generation_strategy( experiment_name=experiment_name, db_settings=self.db_settings) self._experiment = experiment logger.info(f"Loaded {experiment}.") if generation_strategy is None: # pragma: no cover self._generation_strategy = choose_generation_strategy( # pyre-fixme[16]: `Optional` has no attribute `search_space`. search_space=self._experiment.search_space, enforce_sequential_optimization=self. _enforce_sequential_optimization, random_seed=self._random_seed, ) else: self._generation_strategy = generation_strategy logger.info( f"Using generation strategy associated with the loaded experiment:" f" {generation_strategy}.")
def load_experiment_from_database(self, experiment_name: str) -> None: """Load an existing experiment from database using the `DBSettings` passed to this `AxClient` on instantiation. Args: experiment_name: Name of the experiment. Returns: Experiment object. """ if not self.db_settings: raise ValueError( # pragma: no cover "Cannot load an experiment in the absence of the DB settings." "Please initialize `AxClient` with DBSettings.") experiment, generation_strategy = load_experiment_and_generation_strategy( experiment_name=experiment_name, db_settings=self.db_settings) self._experiment = experiment if generation_strategy is None: # pragma: no cover self._generation_strategy = choose_generation_strategy( search_space=self._experiment.search_space, enforce_sequential_optimization=self. _enforce_sequential_optimization, random_seed=self._random_seed, ) else: self._generation_strategy = generation_strategy
def _load_experiment_and_generation_strategy( self, experiment_name: str ) -> Tuple[Optional[Experiment], Optional[GenerationStrategy]]: """Loads experiment and its corresponding generation strategy from database if DB settings are set on this `WithDBSettingsBase` instance. Args: experiment_name: Name of the experiment to load, used as unique identifier by which to find the experiment. Returns: - Tuple of `None` and `None` if `DBSettings` are set and no experiment exists by the given name. - Tuple of `Experiment` and `None` if experiment exists but does not have a generation strategy attached to it. - Tuple of `Experiment` and `GenerationStrategy` if experiment exists and has a generation strategy attached to it. """ if not self.db_settings_set: raise ValueError("Cannot load from DB in absence of DB settings.") try: return load_experiment_and_generation_strategy( experiment_name=experiment_name, db_settings=self.db_settings ) except ValueError: return None, None
def load_experiment(self, experiment_name: str) -> None: """[Work in progress] Load an existing experiment. Args: experiment_name: Name of the experiment. Returns: Experiment object. """ if not self.db_settings: raise ValueError( # pragma: no cover "Cannot load an experiment in the absence of the DB settings." "Please initialize `AxClient` with DBSettings.") experiment, generation_strategy = load_experiment_and_generation_strategy( experiment_name=experiment_name, db_settings=self.db_settings) self._experiment = experiment self.generation_strategy = generation_strategy
def create_experiment( self, parameters: List[Dict[str, Union[TParamValue, List[TParamValue]]]], name: Optional[str] = None, objective_name: Optional[str] = None, minimize: bool = False, parameter_constraints: Optional[List[str]] = None, outcome_constraints: Optional[List[str]] = None, status_quo: Optional[TParameterization] = None, overwrite_existing_experiment: bool = False, experiment_type: Optional[str] = None, choose_generation_strategy_kwargs: Optional[Dict[str, Any]] = None, ) -> None: """Create a new experiment and save it if DBSettings available. Args: parameters: List of dictionaries representing parameters in the experiment search space. Required elements in the dictionaries are: "name" (name of this parameter, string), "type" (type of the parameter: "range", "fixed", or "choice", string), and "bounds" for range parameters (list of two values, lower bound first), "values" for choice parameters (list of values), and "value" for fixed parameters (single value). objective: Name of the metric used as objective in this experiment. This metric must be present in `raw_data` argument to `complete_trial`. name: Name of the experiment to be created. minimize: Whether this experiment represents a minimization problem. parameter_constraints: List of string representation of parameter constraints, such as "x3 >= x4" or "-x3 + 2*x4 - 3.5*x5 >= 2". For the latter constraints, any number of arguments is accepted, and acceptable operators are "<=" and ">=". outcome_constraints: List of string representation of outcome constraints of form "metric_name >= bound", like "m1 <= 3." status_quo: Parameterization of the current state of the system. If set, this will be added to each trial to be evaluated alongside test configurations. overwrite_existing_experiment: If an experiment has already been set on this `AxClient` instance, whether to reset it to the new one. If overwriting the experiment, generation strategy will be re-selected for the new experiment and restarted. To protect experiments in production, one cannot overwrite existing experiments if the experiment is already stored in the database, regardless of the value of `overwrite_existing_experiment`. choose_generation_strategy_kwargs: Keyword arguments to pass to `choose_generation_strategy` function which determines what generation strategy should be used when none was specified on init. """ if self.db_settings and not name: raise ValueError( # pragma: no cover "Must give the experiment a name if `db_settings` is not None." ) if self.db_settings: existing = None try: existing, _ = load_experiment_and_generation_strategy( experiment_name=not_none(name), db_settings=self.db_settings ) except ValueError: # Experiment does not exist, nothing to do. pass if existing: raise ValueError( f"Experiment {name} already exists in the database. " "To protect experiments that are running in production, " "overwriting stored experiments is not allowed. To " "start a new experiment and store it, change the " "experiment's name." ) if self._experiment is not None: if overwrite_existing_experiment: exp_name = self.experiment._name or "untitled" new_exp_name = name or "untitled" logger.info( f"Overwriting existing experiment ({exp_name}) on this client " f"with new experiment ({new_exp_name}) and restarting the " "generation strategy." ) self._generation_strategy = None else: raise ValueError( f"Experiment already created for this client instance. " "Set the `overwrite_existing_experiment` to `True` to overwrite " "with new experiment." ) self._experiment = make_experiment( name=name, parameters=parameters, objective_name=objective_name, minimize=minimize, parameter_constraints=parameter_constraints, outcome_constraints=outcome_constraints, status_quo=status_quo, experiment_type=experiment_type, ) self._set_generation_strategy( choose_generation_strategy_kwargs=choose_generation_strategy_kwargs ) self._save_experiment_to_db_if_possible( suppress_all_errors=self._suppress_storage_errors ) self._save_generation_strategy_to_db_if_possible( suppress_all_errors=self._suppress_storage_errors )
def create_experiment( self, parameters: List[Dict[str, Union[TParamValue, List[TParamValue]]]], name: Optional[str] = None, objective_name: Optional[str] = None, minimize: bool = False, parameter_constraints: Optional[List[str]] = None, outcome_constraints: Optional[List[str]] = None, status_quo: Optional[TParameterization] = None, overwrite_existing_experiment: bool = False, experiment_type: Optional[str] = None, choose_generation_strategy_kwargs: Optional[Dict[str, Any]] = None, ) -> None: """Create a new experiment and save it if DBSettings available. Args: parameters: List of dictionaries representing parameters in the experiment search space. Required elements in the dictionaries are: "name" (name of this parameter, string), "type" (type of the parameter: "range", "fixed", or "choice", string), and "bounds" for range parameters (list of two values, lower bound first), "values" for choice parameters (list of values), and "value" for fixed parameters (single value). objective: Name of the metric used as objective in this experiment. This metric must be present in `raw_data` argument to `complete_trial`. name: Name of the experiment to be created. minimize: Whether this experiment represents a minimization problem. parameter_constraints: List of string representation of parameter constraints, such as "x3 >= x4" or "-x3 + 2*x4 - 3.5*x5 >= 2". For the latter constraints, any number of arguments is accepted, and acceptable operators are "<=" and ">=". outcome_constraints: List of string representation of outcome constraints of form "metric_name >= bound", like "m1 <= 3." status_quo: Parameterization of the current state of the system. If set, this will be added to each trial to be evaluated alongside test configurations. overwrite_existing_experiment: If `DBSettings` were provided on instantiation and the experiment being created has the same name as some experiment already stored, whether to overwrite the existing experiment. Defaults to False. choose_generation_strategy_kwargs: Keyword arguments to pass to `choose_generation_strategy` function which determines what generation strategy should be used when none was specified on init. """ if self.db_settings and not name: raise ValueError( # pragma: no cover "Must give the experiment a name if `db_settings` is not None." ) if self.db_settings: existing = None try: existing, _ = load_experiment_and_generation_strategy( experiment_name=not_none(name), db_settings=self.db_settings) except ValueError: # Experiment does not exist, nothing to do. pass if existing and overwrite_existing_experiment: logger.info(f"Overwriting existing experiment {name}.") elif existing: raise ValueError( f"Experiment {name} exists; set the `overwrite_existing_" "experiment` to `True` to overwrite with new experiment " "or use `ax_client.load_experiment_from_database` to " "continue an existing experiment.") self._experiment = make_experiment( name=name, parameters=parameters, objective_name=objective_name, minimize=minimize, parameter_constraints=parameter_constraints, outcome_constraints=outcome_constraints, status_quo=status_quo, experiment_type=experiment_type, ) if self._generation_strategy is None: self._generation_strategy = choose_generation_strategy( # pyre-fixme[16]: `Optional` has no attribute `search_space`. search_space=self._experiment.search_space, enforce_sequential_optimization=self. _enforce_sequential_optimization, random_seed=self._random_seed, ) self._save_experiment_and_generation_strategy_to_db_if_possible( overwrite_existing_experiment=True)